diff options
Diffstat (limited to '')
63 files changed, 12817 insertions, 0 deletions
diff --git a/src/rgw/services/svc_bi.h b/src/rgw/services/svc_bi.h new file mode 100644 index 000000000..bd811e162 --- /dev/null +++ b/src/rgw/services/svc_bi.h @@ -0,0 +1,44 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +class RGWBucketInfo; +struct RGWBucketEnt; + + +class RGWSI_BucketIndex : public RGWServiceInstance +{ +public: + RGWSI_BucketIndex(CephContext *cct) : RGWServiceInstance(cct) {} + virtual ~RGWSI_BucketIndex() {} + + virtual int init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) = 0; + virtual int clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) = 0; + + virtual int read_stats(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWBucketEnt *stats, + optional_yield y) = 0; + + virtual int handle_overwrite(const DoutPrefixProvider *dpp, + const RGWBucketInfo& info, + const RGWBucketInfo& orig_info, + optional_yield y) = 0; +}; diff --git a/src/rgw/services/svc_bi_rados.cc b/src/rgw/services/svc_bi_rados.cc new file mode 100644 index 000000000..6002b986f --- /dev/null +++ b/src/rgw/services/svc_bi_rados.cc @@ -0,0 +1,509 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_bi_rados.h" +#include "svc_bilog_rados.h" +#include "svc_zone.h" + +#include "rgw_bucket.h" +#include "rgw_zone.h" +#include "rgw_datalog.h" + +#include "cls/rgw/cls_rgw_client.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +static string dir_oid_prefix = ".dir."; + +RGWSI_BucketIndex_RADOS::RGWSI_BucketIndex_RADOS(CephContext *cct) : RGWSI_BucketIndex(cct) +{ +} + +void RGWSI_BucketIndex_RADOS::init(RGWSI_Zone *zone_svc, + RGWSI_RADOS *rados_svc, + RGWSI_BILog_RADOS *bilog_svc, + RGWDataChangesLog *datalog_rados_svc) +{ + svc.zone = zone_svc; + svc.rados = rados_svc; + svc.bilog = bilog_svc; + svc.datalog_rados = datalog_rados_svc; +} + +int RGWSI_BucketIndex_RADOS::open_pool(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + RGWSI_RADOS::Pool *index_pool, + bool mostly_omap) +{ + *index_pool = svc.rados->pool(pool); + return index_pool->open(dpp, RGWSI_RADOS::OpenParams() + .set_mostly_omap(mostly_omap)); +} + +int RGWSI_BucketIndex_RADOS::open_bucket_index_pool(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWSI_RADOS::Pool *index_pool) +{ + const rgw_pool& explicit_pool = bucket_info.bucket.explicit_placement.index_pool; + + if (!explicit_pool.empty()) { + return open_pool(dpp, explicit_pool, index_pool, false); + } + + auto& zonegroup = svc.zone->get_zonegroup(); + auto& zone_params = svc.zone->get_zone_params(); + + const rgw_placement_rule *rule = &bucket_info.placement_rule; + if (rule->empty()) { + rule = &zonegroup.default_placement; + } + auto iter = zone_params.placement_pools.find(rule->name); + if (iter == zone_params.placement_pools.end()) { + ldpp_dout(dpp, 0) << "could not find placement rule " << *rule << " within zonegroup " << dendl; + return -EINVAL; + } + + int r = open_pool(dpp, iter->second.index_pool, index_pool, true); + if (r < 0) + return r; + + return 0; +} + +int RGWSI_BucketIndex_RADOS::open_bucket_index_base(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWSI_RADOS::Pool *index_pool, + string *bucket_oid_base) +{ + const rgw_bucket& bucket = bucket_info.bucket; + int r = open_bucket_index_pool(dpp, bucket_info, index_pool); + if (r < 0) + return r; + + if (bucket.bucket_id.empty()) { + ldpp_dout(dpp, 0) << "ERROR: empty bucket_id for bucket operation" << dendl; + return -EIO; + } + + *bucket_oid_base = dir_oid_prefix; + bucket_oid_base->append(bucket.bucket_id); + + return 0; + +} + +int RGWSI_BucketIndex_RADOS::open_bucket_index(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWSI_RADOS::Pool *index_pool, + string *bucket_oid) +{ + const rgw_bucket& bucket = bucket_info.bucket; + int r = open_bucket_index_pool(dpp, bucket_info, index_pool); + if (r < 0) { + ldpp_dout(dpp, 20) << __func__ << ": open_bucket_index_pool() returned " + << r << dendl; + return r; + } + + if (bucket.bucket_id.empty()) { + ldpp_dout(dpp, 0) << "ERROR: empty bucket id for bucket operation" << dendl; + return -EIO; + } + + *bucket_oid = dir_oid_prefix; + bucket_oid->append(bucket.bucket_id); + + return 0; +} + +static char bucket_obj_with_generation(char *buf, size_t len, const string& bucket_oid_base, uint64_t gen_id, + uint32_t shard_id) +{ + return snprintf(buf, len, "%s.%" PRIu64 ".%d", bucket_oid_base.c_str(), gen_id, shard_id); +} + +static char bucket_obj_without_generation(char *buf, size_t len, const string& bucket_oid_base, uint32_t shard_id) +{ + return snprintf(buf, len, "%s.%d", bucket_oid_base.c_str(), shard_id); +} + +static void get_bucket_index_objects(const string& bucket_oid_base, + uint32_t num_shards, uint64_t gen_id, + map<int, string> *_bucket_objects, + int shard_id = -1) +{ + auto& bucket_objects = *_bucket_objects; + if (!num_shards) { + bucket_objects[0] = bucket_oid_base; + } else { + char buf[bucket_oid_base.size() + 64]; + if (shard_id < 0) { + for (uint32_t i = 0; i < num_shards; ++i) { + if (gen_id) { + bucket_obj_with_generation(buf, sizeof(buf), bucket_oid_base, gen_id, i); + } else { + bucket_obj_without_generation(buf, sizeof(buf), bucket_oid_base, i); + } + bucket_objects[i] = buf; + } + } else { + if (std::cmp_greater(shard_id, num_shards)) { + return; + } else { + if (gen_id) { + bucket_obj_with_generation(buf, sizeof(buf), bucket_oid_base, gen_id, shard_id); + } else { + // for backward compatibility, gen_id(0) will not be added in the object name + bucket_obj_without_generation(buf, sizeof(buf), bucket_oid_base, shard_id); + } + bucket_objects[shard_id] = buf; + } + } + } +} + +static void get_bucket_instance_ids(const RGWBucketInfo& bucket_info, + int num_shards, int shard_id, + map<int, string> *result) +{ + const rgw_bucket& bucket = bucket_info.bucket; + string plain_id = bucket.name + ":" + bucket.bucket_id; + + if (!num_shards) { + (*result)[0] = plain_id; + } else { + char buf[16]; + if (shard_id < 0) { + for (int i = 0; i < num_shards; ++i) { + snprintf(buf, sizeof(buf), ":%d", i); + (*result)[i] = plain_id + buf; + } + } else { + if (shard_id > num_shards) { + return; + } + snprintf(buf, sizeof(buf), ":%d", shard_id); + (*result)[shard_id] = plain_id + buf; + } + } +} + +int RGWSI_BucketIndex_RADOS::open_bucket_index(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + std::optional<int> _shard_id, + const rgw::bucket_index_layout_generation& idx_layout, + RGWSI_RADOS::Pool *index_pool, + map<int, string> *bucket_objs, + map<int, string> *bucket_instance_ids) +{ + int shard_id = _shard_id.value_or(-1); + string bucket_oid_base; + int ret = open_bucket_index_base(dpp, bucket_info, index_pool, &bucket_oid_base); + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << ": open_bucket_index_pool() returned " + << ret << dendl; + return ret; + } + + get_bucket_index_objects(bucket_oid_base, idx_layout.layout.normal.num_shards, + idx_layout.gen, bucket_objs, shard_id); + if (bucket_instance_ids) { + get_bucket_instance_ids(bucket_info, idx_layout.layout.normal.num_shards, + shard_id, bucket_instance_ids); + } + return 0; +} + +void RGWSI_BucketIndex_RADOS::get_bucket_index_object( + const std::string& bucket_oid_base, + const rgw::bucket_index_normal_layout& normal, + uint64_t gen_id, int shard_id, + std::string* bucket_obj) +{ + if (!normal.num_shards) { + // By default with no sharding, we use the bucket oid as itself + (*bucket_obj) = bucket_oid_base; + } else { + char buf[bucket_oid_base.size() + 64]; + if (gen_id) { + bucket_obj_with_generation(buf, sizeof(buf), bucket_oid_base, gen_id, shard_id); + (*bucket_obj) = buf; + ldout(cct, 10) << "bucket_obj is " << (*bucket_obj) << dendl; + } else { + // for backward compatibility, gen_id(0) will not be added in the object name + bucket_obj_without_generation(buf, sizeof(buf), bucket_oid_base, shard_id); + (*bucket_obj) = buf; + } + } +} + +int RGWSI_BucketIndex_RADOS::get_bucket_index_object( + const std::string& bucket_oid_base, + const rgw::bucket_index_normal_layout& normal, + uint64_t gen_id, const std::string& obj_key, + std::string* bucket_obj, int* shard_id) +{ + int r = 0; + switch (normal.hash_type) { + case rgw::BucketHashType::Mod: + if (!normal.num_shards) { + // By default with no sharding, we use the bucket oid as itself + (*bucket_obj) = bucket_oid_base; + if (shard_id) { + *shard_id = -1; + } + } else { + uint32_t sid = bucket_shard_index(obj_key, normal.num_shards); + char buf[bucket_oid_base.size() + 64]; + if (gen_id) { + bucket_obj_with_generation(buf, sizeof(buf), bucket_oid_base, gen_id, sid); + } else { + bucket_obj_without_generation(buf, sizeof(buf), bucket_oid_base, sid); + } + (*bucket_obj) = buf; + if (shard_id) { + *shard_id = (int)sid; + } + } + break; + default: + r = -ENOTSUP; + } + return r; +} + +int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const string& obj_key, + RGWSI_RADOS::Obj *bucket_obj, + int *shard_id) +{ + string bucket_oid_base; + + RGWSI_RADOS::Pool pool; + + int ret = open_bucket_index_base(dpp, bucket_info, &pool, &bucket_oid_base); + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << ": open_bucket_index_pool() returned " + << ret << dendl; + return ret; + } + + string oid; + + const auto& current_index = bucket_info.layout.current_index; + ret = get_bucket_index_object(bucket_oid_base, current_index.layout.normal, + current_index.gen, obj_key, &oid, shard_id); + if (ret < 0) { + ldpp_dout(dpp, 10) << "get_bucket_index_object() returned ret=" << ret << dendl; + return ret; + } + + *bucket_obj = svc.rados->obj(pool, oid); + + return 0; +} + +int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_index_layout_generation& index, + int shard_id, + RGWSI_RADOS::Obj *bucket_obj) +{ + RGWSI_RADOS::Pool index_pool; + string bucket_oid_base; + int ret = open_bucket_index_base(dpp, bucket_info, &index_pool, &bucket_oid_base); + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << ": open_bucket_index_pool() returned " + << ret << dendl; + return ret; + } + + string oid; + + get_bucket_index_object(bucket_oid_base, index.layout.normal, + index.gen, shard_id, &oid); + + *bucket_obj = svc.rados->obj(index_pool, oid); + + return 0; +} + +int RGWSI_BucketIndex_RADOS::cls_bucket_head(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_index_layout_generation& idx_layout, + int shard_id, + vector<rgw_bucket_dir_header> *headers, + map<int, string> *bucket_instance_ids, + optional_yield y) +{ + RGWSI_RADOS::Pool index_pool; + map<int, string> oids; + int r = open_bucket_index(dpp, bucket_info, shard_id, idx_layout, &index_pool, &oids, bucket_instance_ids); + if (r < 0) + return r; + + map<int, struct rgw_cls_list_ret> list_results; + for (auto& iter : oids) { + list_results.emplace(iter.first, rgw_cls_list_ret()); + } + + r = CLSRGWIssueGetDirHeader(index_pool.ioctx(), oids, list_results, cct->_conf->rgw_bucket_index_max_aio)(); + if (r < 0) + return r; + + map<int, struct rgw_cls_list_ret>::iterator iter = list_results.begin(); + for(; iter != list_results.end(); ++iter) { + headers->push_back(std::move(iter->second.dir.header)); + } + return 0; +} + +int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) +{ + RGWSI_RADOS::Pool index_pool; + + string dir_oid = dir_oid_prefix; + int r = open_bucket_index_pool(dpp, bucket_info, &index_pool); + if (r < 0) { + return r; + } + + dir_oid.append(bucket_info.bucket.bucket_id); + + map<int, string> bucket_objs; + get_bucket_index_objects(dir_oid, idx_layout.layout.normal.num_shards, idx_layout.gen, &bucket_objs); + + return CLSRGWIssueBucketIndexInit(index_pool.ioctx(), + bucket_objs, + cct->_conf->rgw_bucket_index_max_aio)(); +} + +int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) +{ + RGWSI_RADOS::Pool index_pool; + + std::string dir_oid = dir_oid_prefix; + int r = open_bucket_index_pool(dpp, bucket_info, &index_pool); + if (r < 0) { + return r; + } + + dir_oid.append(bucket_info.bucket.bucket_id); + + std::map<int, std::string> bucket_objs; + get_bucket_index_objects(dir_oid, idx_layout.layout.normal.num_shards, + idx_layout.gen, &bucket_objs); + + return CLSRGWIssueBucketIndexClean(index_pool.ioctx(), + bucket_objs, + cct->_conf->rgw_bucket_index_max_aio)(); +} + +int RGWSI_BucketIndex_RADOS::read_stats(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWBucketEnt *result, + optional_yield y) +{ + vector<rgw_bucket_dir_header> headers; + + result->bucket = bucket_info.bucket; + int r = cls_bucket_head(dpp, bucket_info, bucket_info.layout.current_index, RGW_NO_SHARD, &headers, nullptr, y); + if (r < 0) { + return r; + } + + result->count = 0; + result->size = 0; + result->size_rounded = 0; + + auto hiter = headers.begin(); + for (; hiter != headers.end(); ++hiter) { + RGWObjCategory category = RGWObjCategory::Main; + auto iter = (hiter->stats).find(category); + if (iter != hiter->stats.end()) { + struct rgw_bucket_category_stats& stats = iter->second; + result->count += stats.num_entries; + result->size += stats.total_size; + result->size_rounded += stats.total_size_rounded; + } + } + + result->placement_rule = std::move(bucket_info.placement_rule); + + return 0; +} + +int RGWSI_BucketIndex_RADOS::get_reshard_status(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, list<cls_rgw_bucket_instance_entry> *status) +{ + map<int, string> bucket_objs; + + RGWSI_RADOS::Pool index_pool; + + int r = open_bucket_index(dpp, bucket_info, + std::nullopt, + bucket_info.layout.current_index, + &index_pool, + &bucket_objs, + nullptr); + if (r < 0) { + return r; + } + + for (auto i : bucket_objs) { + cls_rgw_bucket_instance_entry entry; + + int ret = cls_rgw_get_bucket_resharding(index_pool.ioctx(), i.second, &entry); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, -1) << "ERROR: " << __func__ << ": cls_rgw_get_bucket_resharding() returned ret=" << ret << dendl; + return ret; + } + + status->push_back(entry); + } + + return 0; +} + +int RGWSI_BucketIndex_RADOS::handle_overwrite(const DoutPrefixProvider *dpp, + const RGWBucketInfo& info, + const RGWBucketInfo& orig_info, + optional_yield y) +{ + bool new_sync_enabled = info.datasync_flag_enabled(); + bool old_sync_enabled = orig_info.datasync_flag_enabled(); + + if (old_sync_enabled == new_sync_enabled) { + return 0; // datasync flag didn't change + } + if (info.layout.logs.empty()) { + return 0; // no bilog + } + const auto& bilog = info.layout.logs.back(); + if (bilog.layout.type != rgw::BucketLogType::InIndex) { + return -ENOTSUP; + } + const int shards_num = rgw::num_shards(bilog.layout.in_index); + + int ret; + if (!new_sync_enabled) { + ret = svc.bilog->log_stop(dpp, info, bilog, -1); + } else { + ret = svc.bilog->log_start(dpp, info, bilog, -1); + } + if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed writing bilog (bucket=" << info.bucket << "); ret=" << ret << dendl; + return ret; + } + + for (int i = 0; i < shards_num; ++i) { + ret = svc.datalog_rados->add_entry(dpp, info, bilog, i, y); + if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed writing data log (info.bucket=" << info.bucket << ", shard_id=" << i << ")" << dendl; + } // datalog error is not fatal + } + + return 0; +} diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h new file mode 100644 index 000000000..feba0cfcd --- /dev/null +++ b/src/rgw/services/svc_bi_rados.h @@ -0,0 +1,166 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_datalog.h" +#include "rgw_service.h" +#include "rgw_tools.h" + +#include "svc_bi.h" +#include "svc_rados.h" +#include "svc_tier_rados.h" + +struct rgw_bucket_dir_header; + +class RGWSI_BILog_RADOS; + +#define RGW_NO_SHARD -1 + +#define RGW_SHARDS_PRIME_0 7877 +#define RGW_SHARDS_PRIME_1 65521 + +/* + * Defined Bucket Index Namespaces + */ +#define RGW_OBJ_NS_MULTIPART "multipart" +#define RGW_OBJ_NS_SHADOW "shadow" + +class RGWSI_BucketIndex_RADOS : public RGWSI_BucketIndex +{ + friend class RGWSI_BILog_RADOS; + + int open_pool(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + RGWSI_RADOS::Pool *index_pool, + bool mostly_omap); + + int open_bucket_index_pool(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWSI_RADOS::Pool *index_pool); + int open_bucket_index_base(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWSI_RADOS::Pool *index_pool, + std::string *bucket_oid_base); + + // return the index oid for the given shard id + void get_bucket_index_object(const std::string& bucket_oid_base, + const rgw::bucket_index_normal_layout& normal, + uint64_t gen_id, int shard_id, + std::string* bucket_obj); + // return the index oid and shard id for the given object name + int get_bucket_index_object(const std::string& bucket_oid_base, + const rgw::bucket_index_normal_layout& normal, + uint64_t gen_id, const std::string& obj_key, + std::string* bucket_obj, int* shard_id); + + int cls_bucket_head(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_index_layout_generation& idx_layout, + int shard_id, + std::vector<rgw_bucket_dir_header> *headers, + std::map<int, std::string> *bucket_instance_ids, + optional_yield y); + +public: + + struct Svc { + RGWSI_Zone *zone{nullptr}; + RGWSI_RADOS *rados{nullptr}; + RGWSI_BILog_RADOS *bilog{nullptr}; + RGWDataChangesLog *datalog_rados{nullptr}; + } svc; + + RGWSI_BucketIndex_RADOS(CephContext *cct); + + void init(RGWSI_Zone *zone_svc, + RGWSI_RADOS *rados_svc, + RGWSI_BILog_RADOS *bilog_svc, + RGWDataChangesLog *datalog_rados_svc); + + static int shards_max() { + return RGW_SHARDS_PRIME_1; + } + + static int shard_id(const std::string& key, int max_shards) { + return rgw_shard_id(key, max_shards); + } + + static uint32_t bucket_shard_index(const std::string& key, + int num_shards) { + uint32_t sid = ceph_str_hash_linux(key.c_str(), key.size()); + uint32_t sid2 = sid ^ ((sid & 0xFF) << 24); + return rgw_shards_mod(sid2, num_shards); + } + + static uint32_t bucket_shard_index(const rgw_obj_key& obj_key, + int num_shards) + { + std::string sharding_key; + if (obj_key.ns == RGW_OBJ_NS_MULTIPART) { + RGWMPObj mp; + mp.from_meta(obj_key.name); + sharding_key = mp.get_key(); + } else { + sharding_key = obj_key.name; + } + + return bucket_shard_index(sharding_key, num_shards); + } + + int init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info,const rgw::bucket_index_layout_generation& idx_layout) override; + int clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout) override; + + /* RADOS specific */ + + int read_stats(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWBucketEnt *stats, + optional_yield y) override; + + int get_reshard_status(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, + std::list<cls_rgw_bucket_instance_entry> *status); + + int handle_overwrite(const DoutPrefixProvider *dpp, const RGWBucketInfo& info, + const RGWBucketInfo& orig_info, + optional_yield y) override; + + int open_bucket_index_shard(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const std::string& obj_key, + RGWSI_RADOS::Obj *bucket_obj, + int *shard_id); + + int open_bucket_index_shard(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_index_layout_generation& index, + int shard_id, RGWSI_RADOS::Obj *bucket_obj); + + int open_bucket_index(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + RGWSI_RADOS::Pool *index_pool, + std::string *bucket_oid); + + int open_bucket_index(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + std::optional<int> shard_id, + const rgw::bucket_index_layout_generation& idx_layout, + RGWSI_RADOS::Pool *index_pool, + std::map<int, std::string> *bucket_objs, + std::map<int, std::string> *bucket_instance_ids); +}; + + diff --git a/src/rgw/services/svc_bilog_rados.cc b/src/rgw/services/svc_bilog_rados.cc new file mode 100644 index 000000000..f4bb13ec1 --- /dev/null +++ b/src/rgw/services/svc_bilog_rados.cc @@ -0,0 +1,220 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_bilog_rados.h" +#include "svc_bi_rados.h" + +#include "cls/rgw/cls_rgw_client.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +RGWSI_BILog_RADOS::RGWSI_BILog_RADOS(CephContext *cct) : RGWServiceInstance(cct) +{ +} + +void RGWSI_BILog_RADOS::init(RGWSI_BucketIndex_RADOS *bi_rados_svc) +{ + svc.bi = bi_rados_svc; +} + +int RGWSI_BILog_RADOS::log_trim(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_log_layout_generation& log_layout, + int shard_id, + std::string_view start_marker, + std::string_view end_marker) +{ + RGWSI_RADOS::Pool index_pool; + map<int, string> bucket_objs; + + BucketIndexShardsManager start_marker_mgr; + BucketIndexShardsManager end_marker_mgr; + + const auto& current_index = rgw::log_to_index_layout(log_layout); + int r = svc.bi->open_bucket_index(dpp, bucket_info, shard_id, current_index, &index_pool, &bucket_objs, nullptr); + if (r < 0) { + return r; + } + + r = start_marker_mgr.from_string(start_marker, shard_id); + if (r < 0) { + return r; + } + + r = end_marker_mgr.from_string(end_marker, shard_id); + if (r < 0) { + return r; + } + + return CLSRGWIssueBILogTrim(index_pool.ioctx(), start_marker_mgr, end_marker_mgr, bucket_objs, + cct->_conf->rgw_bucket_index_max_aio)(); +} + +int RGWSI_BILog_RADOS::log_start(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_log_layout_generation& log_layout, int shard_id) +{ + RGWSI_RADOS::Pool index_pool; + map<int, string> bucket_objs; + const auto& current_index = rgw::log_to_index_layout(log_layout); + int r = svc.bi->open_bucket_index(dpp, bucket_info, shard_id, current_index, &index_pool, &bucket_objs, nullptr); + if (r < 0) + return r; + + return CLSRGWIssueResyncBucketBILog(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); +} + +int RGWSI_BILog_RADOS::log_stop(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_log_layout_generation& log_layout, int shard_id) +{ + RGWSI_RADOS::Pool index_pool; + map<int, string> bucket_objs; + const auto& current_index = rgw::log_to_index_layout(log_layout); + int r = svc.bi->open_bucket_index(dpp, bucket_info, shard_id, current_index, &index_pool, &bucket_objs, nullptr); + if (r < 0) + return r; + + return CLSRGWIssueBucketBILogStop(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); +} + +static void build_bucket_index_marker(const string& shard_id_str, + const string& shard_marker, + string *marker) { + if (marker) { + *marker = shard_id_str; + marker->append(BucketIndexShardsManager::KEY_VALUE_SEPARATOR); + marker->append(shard_marker); + } +} + +int RGWSI_BILog_RADOS::log_list(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_log_layout_generation& log_layout, + int shard_id, string& marker, uint32_t max, + std::list<rgw_bi_log_entry>& result, bool *truncated) +{ + ldpp_dout(dpp, 20) << __func__ << ": " << bucket_info.bucket << " marker " << marker << " shard_id=" << shard_id << " max " << max << dendl; + result.clear(); + + RGWSI_RADOS::Pool index_pool; + map<int, string> oids; + map<int, cls_rgw_bi_log_list_ret> bi_log_lists; + const auto& current_index = rgw::log_to_index_layout(log_layout); + int r = svc.bi->open_bucket_index(dpp, bucket_info, shard_id, current_index, &index_pool, &oids, nullptr); + if (r < 0) + return r; + + BucketIndexShardsManager marker_mgr; + bool has_shards = (oids.size() > 1 || shard_id >= 0); + // If there are multiple shards for the bucket index object, the marker + // should have the pattern '{shard_id_1}#{shard_marker_1},{shard_id_2}# + // {shard_marker_2}...', if there is no sharding, the bi_log_list should + // only contain one record, and the key is the bucket instance id. + r = marker_mgr.from_string(marker, shard_id); + if (r < 0) + return r; + + r = CLSRGWIssueBILogList(index_pool.ioctx(), marker_mgr, max, oids, bi_log_lists, cct->_conf->rgw_bucket_index_max_aio)(); + if (r < 0) + return r; + + map<int, list<rgw_bi_log_entry>::iterator> vcurrents; + map<int, list<rgw_bi_log_entry>::iterator> vends; + if (truncated) { + *truncated = false; + } + map<int, cls_rgw_bi_log_list_ret>::iterator miter = bi_log_lists.begin(); + for (; miter != bi_log_lists.end(); ++miter) { + int shard_id = miter->first; + vcurrents[shard_id] = miter->second.entries.begin(); + vends[shard_id] = miter->second.entries.end(); + if (truncated) { + *truncated = (*truncated || miter->second.truncated); + } + } + + size_t total = 0; + bool has_more = true; + map<int, list<rgw_bi_log_entry>::iterator>::iterator viter; + map<int, list<rgw_bi_log_entry>::iterator>::iterator eiter; + while (total < max && has_more) { + has_more = false; + + viter = vcurrents.begin(); + eiter = vends.begin(); + + for (; total < max && viter != vcurrents.end(); ++viter, ++eiter) { + assert (eiter != vends.end()); + + int shard_id = viter->first; + list<rgw_bi_log_entry>::iterator& liter = viter->second; + + if (liter == eiter->second){ + continue; + } + rgw_bi_log_entry& entry = *(liter); + if (has_shards) { + char buf[16]; + snprintf(buf, sizeof(buf), "%d", shard_id); + string tmp_id; + build_bucket_index_marker(buf, entry.id, &tmp_id); + entry.id.swap(tmp_id); + } + marker_mgr.add(shard_id, entry.id); + result.push_back(entry); + total++; + has_more = true; + ++liter; + } + } + + if (truncated) { + for (viter = vcurrents.begin(), eiter = vends.begin(); viter != vcurrents.end(); ++viter, ++eiter) { + assert (eiter != vends.end()); + *truncated = (*truncated || (viter->second != eiter->second)); + } + } + + // Refresh marker, if there are multiple shards, the output will look like + // '{shard_oid_1}#{shard_marker_1},{shard_oid_2}#{shard_marker_2}...', + // if there is no sharding, the simply marker (without oid) is returned + if (has_shards) { + marker_mgr.to_string(&marker); + } else { + if (!result.empty()) { + marker = result.rbegin()->id; + } + } + + return 0; +} + +int RGWSI_BILog_RADOS::get_log_status(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_log_layout_generation& log_layout, + int shard_id, + map<int, string> *markers, + optional_yield y) +{ + vector<rgw_bucket_dir_header> headers; + map<int, string> bucket_instance_ids; + const auto& current_index = rgw::log_to_index_layout(log_layout); + int r = svc.bi->cls_bucket_head(dpp, bucket_info, current_index, shard_id, &headers, &bucket_instance_ids, y); + if (r < 0) + return r; + + ceph_assert(headers.size() == bucket_instance_ids.size()); + + auto iter = headers.begin(); + map<int, string>::iterator viter = bucket_instance_ids.begin(); + + for(; iter != headers.end(); ++iter, ++viter) { + if (shard_id >= 0) { + (*markers)[shard_id] = iter->max_marker; + } else { + (*markers)[viter->first] = iter->max_marker; + } + } + + return 0; +} + diff --git a/src/rgw/services/svc_bilog_rados.h b/src/rgw/services/svc_bilog_rados.h new file mode 100644 index 000000000..e9d5dbb5c --- /dev/null +++ b/src/rgw/services/svc_bilog_rados.h @@ -0,0 +1,60 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_rados.h" + + + + +class RGWSI_BILog_RADOS : public RGWServiceInstance +{ +public: + struct Svc { + RGWSI_BucketIndex_RADOS *bi{nullptr}; + } svc; + + RGWSI_BILog_RADOS(CephContext *cct); + + void init(RGWSI_BucketIndex_RADOS *bi_rados_svc); + + int log_start(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_log_layout_generation& log_layout, int shard_id); + int log_stop(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_log_layout_generation& log_layout, int shard_id); + + int log_trim(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, + const rgw::bucket_log_layout_generation& log_layout, + int shard_id, + std::string_view start_marker, + std::string_view end_marker); + int log_list(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, + const rgw::bucket_log_layout_generation& log_layout, + int shard_id, + std::string& marker, + uint32_t max, + std::list<rgw_bi_log_entry>& result, + bool *truncated); + + int get_log_status(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + const rgw::bucket_log_layout_generation& log_layout, + int shard_id, + std::map<int, std::string> *markers, + optional_yield y); +}; diff --git a/src/rgw/services/svc_bucket.cc b/src/rgw/services/svc_bucket.cc new file mode 100644 index 000000000..b115990d2 --- /dev/null +++ b/src/rgw/services/svc_bucket.cc @@ -0,0 +1,25 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + + +#include "svc_bucket.h" + +#define dout_subsys ceph_subsys_rgw + +std::string RGWSI_Bucket::get_entrypoint_meta_key(const rgw_bucket& bucket) +{ + if (bucket.bucket_id.empty()) { + return bucket.get_key(); + } + + rgw_bucket b(bucket); + b.bucket_id.clear(); + + return b.get_key(); +} + +std::string RGWSI_Bucket::get_bi_meta_key(const rgw_bucket& bucket) +{ + return bucket.get_key(); +} + diff --git a/src/rgw/services/svc_bucket.h b/src/rgw/services/svc_bucket.h new file mode 100644 index 000000000..4a526e4f2 --- /dev/null +++ b/src/rgw/services/svc_bucket.h @@ -0,0 +1,111 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_bucket_types.h" + +class RGWSI_Bucket : public RGWServiceInstance +{ +public: + RGWSI_Bucket(CephContext *cct) : RGWServiceInstance(cct) {} + virtual ~RGWSI_Bucket() {} + + static std::string get_entrypoint_meta_key(const rgw_bucket& bucket); + static std::string get_bi_meta_key(const rgw_bucket& bucket); + + virtual RGWSI_Bucket_BE_Handler& get_ep_be_handler() = 0; + virtual RGWSI_BucketInstance_BE_Handler& get_bi_be_handler() = 0; + + virtual int read_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const std::string& key, + RGWBucketEntryPoint *entry_point, + RGWObjVersionTracker *objv_tracker, + real_time *pmtime, + std::map<std::string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp, + rgw_cache_entry_info *cache_info = nullptr, + boost::optional<obj_version> refresh_version = boost::none) = 0; + + virtual int store_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const std::string& key, + RGWBucketEntryPoint& info, + bool exclusive, + real_time mtime, + std::map<std::string, bufferlist> *pattrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int remove_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const std::string& key, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int read_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const std::string& key, + RGWBucketInfo *info, + real_time *pmtime, + std::map<std::string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp, + rgw_cache_entry_info *cache_info = nullptr, + boost::optional<obj_version> refresh_version = boost::none) = 0; + + virtual int read_bucket_info(RGWSI_Bucket_X_Ctx& ep_ctx, + const rgw_bucket& bucket, + RGWBucketInfo *info, + real_time *pmtime, + std::map<std::string, bufferlist> *pattrs, + boost::optional<obj_version> refresh_version, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const std::string& key, + RGWBucketInfo& info, + std::optional<RGWBucketInfo *> orig_info, /* nullopt: orig_info was not fetched, + nullptr: orig_info was not found (new bucket instance */ + bool exclusive, + real_time mtime, + std::map<std::string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int remove_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const std::string& key, + const RGWBucketInfo& bucket_info, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int read_bucket_stats(RGWSI_Bucket_X_Ctx& ctx, + const rgw_bucket& bucket, + RGWBucketEnt *ent, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int read_buckets_stats(RGWSI_Bucket_X_Ctx& ctx, + std::map<std::string, RGWBucketEnt>& m, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; +}; + diff --git a/src/rgw/services/svc_bucket_sobj.cc b/src/rgw/services/svc_bucket_sobj.cc new file mode 100644 index 000000000..08a528015 --- /dev/null +++ b/src/rgw/services/svc_bucket_sobj.cc @@ -0,0 +1,644 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + + +#include "svc_bucket_sobj.h" +#include "svc_zone.h" +#include "svc_sys_obj.h" +#include "svc_sys_obj_cache.h" +#include "svc_bi.h" +#include "svc_meta.h" +#include "svc_meta_be_sobj.h" +#include "svc_sync_modules.h" + +#include "rgw_bucket.h" +#include "rgw_tools.h" +#include "rgw_zone.h" + +#define dout_subsys ceph_subsys_rgw + +#define RGW_BUCKET_INSTANCE_MD_PREFIX ".bucket.meta." + +using namespace std; + +class RGWSI_Bucket_SObj_Module : public RGWSI_MBSObj_Handler_Module { + RGWSI_Bucket_SObj::Svc& svc; + + const string prefix; +public: + RGWSI_Bucket_SObj_Module(RGWSI_Bucket_SObj::Svc& _svc) : RGWSI_MBSObj_Handler_Module("bucket"), + svc(_svc) {} + + void get_pool_and_oid(const string& key, rgw_pool *pool, string *oid) override { + if (pool) { + *pool = svc.zone->get_zone_params().domain_root; + } + if (oid) { + *oid = key; + } + } + + const string& get_oid_prefix() override { + return prefix; + } + + bool is_valid_oid(const string& oid) override { + return (!oid.empty() && oid[0] != '.'); + } + + string key_to_oid(const string& key) override { + return key; + } + + string oid_to_key(const string& oid) override { + /* should have been called after is_valid_oid(), + * so no need to check for validity */ + return oid; + } +}; + +class RGWSI_BucketInstance_SObj_Module : public RGWSI_MBSObj_Handler_Module { + RGWSI_Bucket_SObj::Svc& svc; + + const string prefix; +public: + RGWSI_BucketInstance_SObj_Module(RGWSI_Bucket_SObj::Svc& _svc) : RGWSI_MBSObj_Handler_Module("bucket.instance"), + svc(_svc), prefix(RGW_BUCKET_INSTANCE_MD_PREFIX) {} + + void get_pool_and_oid(const string& key, rgw_pool *pool, string *oid) override { + if (pool) { + *pool = svc.zone->get_zone_params().domain_root; + } + if (oid) { + *oid = key_to_oid(key); + } + } + + const string& get_oid_prefix() override { + return prefix; + } + + bool is_valid_oid(const string& oid) override { + return (oid.compare(0, prefix.size(), RGW_BUCKET_INSTANCE_MD_PREFIX) == 0); + } + +// 'tenant/' is used in bucket instance keys for sync to avoid parsing ambiguity +// with the existing instance[:shard] format. once we parse the shard, the / is +// replaced with a : to match the [tenant:]instance format + string key_to_oid(const string& key) override { + string oid = prefix + key; + + // replace tenant/ with tenant: + auto c = oid.find('/', prefix.size()); + if (c != string::npos) { + oid[c] = ':'; + } + + return oid; + } + + // convert bucket instance oids back to the tenant/ format for metadata keys. + // it's safe to parse 'tenant:' only for oids, because they won't contain the + // optional :shard at the end + string oid_to_key(const string& oid) override { + /* this should have been called after oid was checked for validity */ + + if (oid.size() < prefix.size()) { /* just sanity check */ + return string(); + } + + string key = oid.substr(prefix.size()); + + // find first : (could be tenant:bucket or bucket:instance) + auto c = key.find(':'); + if (c != string::npos) { + // if we find another :, the first one was for tenant + if (key.find(':', c + 1) != string::npos) { + key[c] = '/'; + } + } + + return key; + } + + /* + * hash entry for mdlog placement. Use the same hash key we'd have for the bucket entry + * point, so that the log entries end up at the same log shard, so that we process them + * in order + */ + string get_hash_key(const string& key) override { + string k = "bucket:"; + int pos = key.find(':'); + if (pos < 0) + k.append(key); + else + k.append(key.substr(0, pos)); + + return k; + } +}; + +RGWSI_Bucket_SObj::RGWSI_Bucket_SObj(CephContext *cct): RGWSI_Bucket(cct) { +} + +RGWSI_Bucket_SObj::~RGWSI_Bucket_SObj() { +} + +void RGWSI_Bucket_SObj::init(RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, + RGWSI_SysObj_Cache *_cache_svc, RGWSI_BucketIndex *_bi, + RGWSI_Meta *_meta_svc, RGWSI_MetaBackend *_meta_be_svc, + RGWSI_SyncModules *_sync_modules_svc, + RGWSI_Bucket_Sync *_bucket_sync_svc) +{ + svc.bucket = this; + svc.zone = _zone_svc; + svc.sysobj = _sysobj_svc; + svc.cache = _cache_svc; + svc.bi = _bi; + svc.meta = _meta_svc; + svc.meta_be = _meta_be_svc; + svc.sync_modules = _sync_modules_svc; + svc.bucket_sync = _bucket_sync_svc; +} + +int RGWSI_Bucket_SObj::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + binfo_cache.reset(new RGWChainedCacheImpl<bucket_info_cache_entry>); + binfo_cache->init(svc.cache); + + /* create first backend handler for bucket entrypoints */ + + RGWSI_MetaBackend_Handler *ep_handler; + + int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &ep_handler); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to create be handler: r=" << r << dendl; + return r; + } + + ep_be_handler = ep_handler; + + RGWSI_MetaBackend_Handler_SObj *ep_bh = static_cast<RGWSI_MetaBackend_Handler_SObj *>(ep_handler); + + auto ep_module = new RGWSI_Bucket_SObj_Module(svc); + ep_be_module.reset(ep_module); + ep_bh->set_module(ep_module); + + /* create a second backend handler for bucket instance */ + + RGWSI_MetaBackend_Handler *bi_handler; + + r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &bi_handler); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to create be handler: r=" << r << dendl; + return r; + } + + bi_be_handler = bi_handler; + + RGWSI_MetaBackend_Handler_SObj *bi_bh = static_cast<RGWSI_MetaBackend_Handler_SObj *>(bi_handler); + + auto bi_module = new RGWSI_BucketInstance_SObj_Module(svc); + bi_be_module.reset(bi_module); + bi_bh->set_module(bi_module); + + return 0; +} + +int RGWSI_Bucket_SObj::read_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const string& key, + RGWBucketEntryPoint *entry_point, + RGWObjVersionTracker *objv_tracker, + real_time *pmtime, + map<string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version> refresh_version) +{ + bufferlist bl; + + auto params = RGWSI_MBSObj_GetParams(&bl, pattrs, pmtime).set_cache_info(cache_info) + .set_refresh_version(refresh_version); + + int ret = svc.meta_be->get_entry(ctx.get(), key, params, objv_tracker, y, dpp); + if (ret < 0) { + return ret; + } + + auto iter = bl.cbegin(); + try { + decode(*entry_point, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: could not decode buffer info, caught buffer::error" << dendl; + return -EIO; + } + return 0; +} + +int RGWSI_Bucket_SObj::store_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const string& key, + RGWBucketEntryPoint& info, + bool exclusive, + real_time mtime, + map<string, bufferlist> *pattrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + bufferlist bl; + encode(info, bl); + + RGWSI_MBSObj_PutParams params(bl, pattrs, mtime, exclusive); + + int ret = svc.meta_be->put(ctx.get(), key, params, objv_tracker, y, dpp); + if (ret < 0) { + return ret; + } + + return ret; +} + +int RGWSI_Bucket_SObj::remove_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const string& key, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + RGWSI_MBSObj_RemoveParams params; + return svc.meta_be->remove(ctx.get(), key, params, objv_tracker, y, dpp); +} + +int RGWSI_Bucket_SObj::read_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const string& key, + RGWBucketInfo *info, + real_time *pmtime, map<string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version> refresh_version) +{ + string cache_key("bi/"); + cache_key.append(key); + + if (auto e = binfo_cache->find(cache_key)) { + if (refresh_version && + e->info.objv_tracker.read_version.compare(&(*refresh_version))) { + ldpp_dout(dpp, -1) << "WARNING: The bucket info cache is inconsistent. This is " + << "a failure that should be debugged. I am a nice machine, " + << "so I will try to recover." << dendl; + binfo_cache->invalidate(key); + } else { + *info = e->info; + if (pattrs) + *pattrs = e->attrs; + if (pmtime) + *pmtime = e->mtime; + return 0; + } + } + + bucket_info_cache_entry e; + rgw_cache_entry_info ci; + + int ret = do_read_bucket_instance_info(ctx, key, + &e.info, &e.mtime, &e.attrs, + &ci, refresh_version, y, dpp); + *info = e.info; + + if (ret < 0) { + if (ret != -ENOENT) { + ldpp_dout(dpp, -1) << "ERROR: do_read_bucket_instance_info failed: " << ret << dendl; + } else { + ldpp_dout(dpp, 20) << "do_read_bucket_instance_info, bucket instance not found (key=" << key << ")" << dendl; + } + return ret; + } + + if (pmtime) { + *pmtime = e.mtime; + } + if (pattrs) { + *pattrs = e.attrs; + } + if (cache_info) { + *cache_info = ci; + } + + /* chain to only bucket instance and *not* bucket entrypoint */ + if (!binfo_cache->put(dpp, svc.cache, cache_key, &e, {&ci})) { + ldpp_dout(dpp, 20) << "couldn't put binfo cache entry, might have raced with data changes" << dendl; + } + + if (refresh_version && + refresh_version->compare(&info->objv_tracker.read_version)) { + ldpp_dout(dpp, -1) << "WARNING: The OSD has the same version I have. Something may " + << "have gone squirrelly. An administrator may have forced a " + << "change; otherwise there is a problem somewhere." << dendl; + } + + return 0; +} + +int RGWSI_Bucket_SObj::do_read_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const string& key, + RGWBucketInfo *info, + real_time *pmtime, map<string, bufferlist> *pattrs, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version> refresh_version, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + bufferlist bl; + RGWObjVersionTracker ot; + + auto params = RGWSI_MBSObj_GetParams(&bl, pattrs, pmtime).set_cache_info(cache_info) + .set_refresh_version(refresh_version); + + int ret = svc.meta_be->get_entry(ctx.get(), key, params, &ot, y, dpp); + if (ret < 0) { + return ret; + } + + auto iter = bl.cbegin(); + try { + decode(*info, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: could not decode buffer info, caught buffer::error" << dendl; + return -EIO; + } + info->objv_tracker = ot; + return 0; +} + +int RGWSI_Bucket_SObj::read_bucket_info(RGWSI_Bucket_X_Ctx& ctx, + const rgw_bucket& bucket, + RGWBucketInfo *info, + real_time *pmtime, + map<string, bufferlist> *pattrs, + boost::optional<obj_version> refresh_version, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + rgw_cache_entry_info cache_info; + + if (!bucket.bucket_id.empty()) { + return read_bucket_instance_info(ctx.bi, get_bi_meta_key(bucket), + info, + pmtime, pattrs, + y, + dpp, + &cache_info, refresh_version); + } + + string bucket_entry = get_entrypoint_meta_key(bucket); + string cache_key("b/"); + cache_key.append(bucket_entry); + + if (auto e = binfo_cache->find(cache_key)) { + bool found_version = (bucket.bucket_id.empty() || + bucket.bucket_id == e->info.bucket.bucket_id); + + if (!found_version || + (refresh_version && + e->info.objv_tracker.read_version.compare(&(*refresh_version)))) { + ldpp_dout(dpp, -1) << "WARNING: The bucket info cache is inconsistent. This is " + << "a failure that should be debugged. I am a nice machine, " + << "so I will try to recover." << dendl; + binfo_cache->invalidate(cache_key); + } else { + *info = e->info; + if (pattrs) + *pattrs = e->attrs; + if (pmtime) + *pmtime = e->mtime; + return 0; + } + } + + RGWBucketEntryPoint entry_point; + real_time ep_mtime; + RGWObjVersionTracker ot; + rgw_cache_entry_info entry_cache_info; + int ret = read_bucket_entrypoint_info(ctx.ep, bucket_entry, + &entry_point, &ot, &ep_mtime, pattrs, + y, + dpp, + &entry_cache_info, refresh_version); + if (ret < 0) { + /* only init these fields */ + info->bucket = bucket; + return ret; + } + + if (entry_point.has_bucket_info) { + *info = entry_point.old_bucket_info; + info->bucket.tenant = bucket.tenant; + ldpp_dout(dpp, 20) << "rgw_get_bucket_info: old bucket info, bucket=" << info->bucket << " owner " << info->owner << dendl; + return 0; + } + + /* data is in the bucket instance object, we need to get attributes from there, clear everything + * that we got + */ + if (pattrs) { + pattrs->clear(); + } + + ldpp_dout(dpp, 20) << "rgw_get_bucket_info: bucket instance: " << entry_point.bucket << dendl; + + + /* read bucket instance info */ + + bucket_info_cache_entry e; + + ret = read_bucket_instance_info(ctx.bi, get_bi_meta_key(entry_point.bucket), + &e.info, &e.mtime, &e.attrs, + y, + dpp, + &cache_info, refresh_version); + *info = e.info; + if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR: read_bucket_instance_from_oid failed: " << ret << dendl; + info->bucket = bucket; + // XXX and why return anything in case of an error anyway? + return ret; + } + + if (pmtime) + *pmtime = e.mtime; + if (pattrs) + *pattrs = e.attrs; + + /* chain to both bucket entry point and bucket instance */ + if (!binfo_cache->put(dpp, svc.cache, cache_key, &e, {&entry_cache_info, &cache_info})) { + ldpp_dout(dpp, 20) << "couldn't put binfo cache entry, might have raced with data changes" << dendl; + } + + if (refresh_version && + refresh_version->compare(&info->objv_tracker.read_version)) { + ldpp_dout(dpp, -1) << "WARNING: The OSD has the same version I have. Something may " + << "have gone squirrelly. An administrator may have forced a " + << "change; otherwise there is a problem somewhere." << dendl; + } + + return 0; +} + + +int RGWSI_Bucket_SObj::store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const string& key, + RGWBucketInfo& info, + std::optional<RGWBucketInfo *> orig_info, + bool exclusive, + real_time mtime, + map<string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + bufferlist bl; + encode(info, bl); + + /* + * we might need some special handling if overwriting + */ + RGWBucketInfo shared_bucket_info; + if (!orig_info && !exclusive) { /* if exclusive, we're going to fail when try + to overwrite, so the whole check here is moot */ + /* + * we're here because orig_info wasn't passed in + * we don't have info about what was there before, so need to fetch first + */ + int r = read_bucket_instance_info(ctx, + key, + &shared_bucket_info, + nullptr, nullptr, + y, + dpp, + nullptr, boost::none); + if (r < 0) { + if (r != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): read_bucket_instance_info() of key=" << key << " returned r=" << r << dendl; + return r; + } + } else { + orig_info = &shared_bucket_info; + } + } + + if (orig_info && *orig_info && !exclusive) { + int r = svc.bi->handle_overwrite(dpp, info, *(orig_info.value()), y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): svc.bi->handle_overwrite() of key=" << key << " returned r=" << r << dendl; + return r; + } + } + + RGWSI_MBSObj_PutParams params(bl, pattrs, mtime, exclusive); + + int ret = svc.meta_be->put(ctx.get(), key, params, &info.objv_tracker, y, dpp); + + if (ret >= 0) { + int r = svc.bucket_sync->handle_bi_update(dpp, info, + orig_info.value_or(nullptr), + y); + if (r < 0) { + return r; + } + } else if (ret == -EEXIST) { + /* well, if it's exclusive we shouldn't overwrite it, because we might race with another + * bucket operation on this specific bucket (e.g., being synced from the master), but + * since bucket instance meta object is unique for this specific bucket instance, we don't + * need to return an error. + * A scenario where we'd get -EEXIST here, is in a multi-zone config, we're not on the + * master, creating a bucket, sending bucket creation to the master, we create the bucket + * locally, while in the sync thread we sync the new bucket. + */ + ret = 0; + } + + if (ret < 0) { + return ret; + } + + return ret; +} + +int RGWSI_Bucket_SObj::remove_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const string& key, + const RGWBucketInfo& info, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + RGWSI_MBSObj_RemoveParams params; + int ret = svc.meta_be->remove_entry(dpp, ctx.get(), key, params, objv_tracker, y); + + if (ret < 0 && + ret != -ENOENT) { + return ret; + } + + int r = svc.bucket_sync->handle_bi_removal(dpp, info, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update bucket instance sync index: r=" << r << dendl; + /* returning success as index is just keeping hints, so will keep extra hints, + * but bucket removal succeeded + */ + } + + return 0; +} + +int RGWSI_Bucket_SObj::read_bucket_stats(const RGWBucketInfo& bucket_info, + RGWBucketEnt *ent, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + ent->count = 0; + ent->size = 0; + ent->size_rounded = 0; + + vector<rgw_bucket_dir_header> headers; + + int r = svc.bi->read_stats(dpp, bucket_info, ent, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): read_stats returned r=" << r << dendl; + return r; + } + + return 0; +} + +int RGWSI_Bucket_SObj::read_bucket_stats(RGWSI_Bucket_X_Ctx& ctx, + const rgw_bucket& bucket, + RGWBucketEnt *ent, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + RGWBucketInfo bucket_info; + int ret = read_bucket_info(ctx, bucket, &bucket_info, nullptr, nullptr, boost::none, y, dpp); + if (ret < 0) { + return ret; + } + + return read_bucket_stats(bucket_info, ent, y, dpp); +} + +int RGWSI_Bucket_SObj::read_buckets_stats(RGWSI_Bucket_X_Ctx& ctx, + map<string, RGWBucketEnt>& m, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + map<string, RGWBucketEnt>::iterator iter; + for (iter = m.begin(); iter != m.end(); ++iter) { + RGWBucketEnt& ent = iter->second; + int r = read_bucket_stats(ctx, ent.bucket, &ent, y, dpp); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): read_bucket_stats returned r=" << r << dendl; + return r; + } + } + + return m.size(); +} diff --git a/src/rgw/services/svc_bucket_sobj.h b/src/rgw/services/svc_bucket_sobj.h new file mode 100644 index 000000000..8e9fe063c --- /dev/null +++ b/src/rgw/services/svc_bucket_sobj.h @@ -0,0 +1,180 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_meta_be.h" +#include "svc_bucket_types.h" +#include "svc_bucket.h" +#include "svc_bucket_sync.h" + +class RGWSI_Zone; +class RGWSI_SysObj; +class RGWSI_SysObj_Cache; +class RGWSI_Meta; +class RGWSI_SyncModules; + +struct rgw_cache_entry_info; + +template <class T> +class RGWChainedCacheImpl; + +class RGWSI_Bucket_SObj : public RGWSI_Bucket +{ + struct bucket_info_cache_entry { + RGWBucketInfo info; + real_time mtime; + std::map<std::string, bufferlist> attrs; + }; + + using RGWChainedCacheImpl_bucket_info_cache_entry = RGWChainedCacheImpl<bucket_info_cache_entry>; + std::unique_ptr<RGWChainedCacheImpl_bucket_info_cache_entry> binfo_cache; + + RGWSI_Bucket_BE_Handler ep_be_handler; + std::unique_ptr<RGWSI_MetaBackend::Module> ep_be_module; + RGWSI_BucketInstance_BE_Handler bi_be_handler; + std::unique_ptr<RGWSI_MetaBackend::Module> bi_be_module; + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + + int do_read_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const std::string& key, + RGWBucketInfo *info, + real_time *pmtime, + std::map<std::string, bufferlist> *pattrs, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version> refresh_version, + optional_yield y, + const DoutPrefixProvider *dpp); + + int read_bucket_stats(const RGWBucketInfo& bucket_info, + RGWBucketEnt *ent, + optional_yield y, + const DoutPrefixProvider *dpp); + +public: + struct Svc { + RGWSI_Bucket_SObj *bucket{nullptr}; + RGWSI_BucketIndex *bi{nullptr}; + RGWSI_Zone *zone{nullptr}; + RGWSI_SysObj *sysobj{nullptr}; + RGWSI_SysObj_Cache *cache{nullptr}; + RGWSI_Meta *meta{nullptr}; + RGWSI_MetaBackend *meta_be{nullptr}; + RGWSI_SyncModules *sync_modules{nullptr}; + RGWSI_Bucket_Sync *bucket_sync{nullptr}; + } svc; + + RGWSI_Bucket_SObj(CephContext *cct); + ~RGWSI_Bucket_SObj(); + + RGWSI_Bucket_BE_Handler& get_ep_be_handler() override { + return ep_be_handler; + } + + RGWSI_BucketInstance_BE_Handler& get_bi_be_handler() override { + return bi_be_handler; + } + + void init(RGWSI_Zone *_zone_svc, + RGWSI_SysObj *_sysobj_svc, + RGWSI_SysObj_Cache *_cache_svc, + RGWSI_BucketIndex *_bi, + RGWSI_Meta *_meta_svc, + RGWSI_MetaBackend *_meta_be_svc, + RGWSI_SyncModules *_sync_modules_svc, + RGWSI_Bucket_Sync *_bucket_sync_svc); + + + int read_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const std::string& key, + RGWBucketEntryPoint *entry_point, + RGWObjVersionTracker *objv_tracker, + real_time *pmtime, + std::map<std::string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp, + rgw_cache_entry_info *cache_info = nullptr, + boost::optional<obj_version> refresh_version = boost::none) override; + + int store_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const std::string& key, + RGWBucketEntryPoint& info, + bool exclusive, + real_time mtime, + std::map<std::string, bufferlist> *pattrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int remove_bucket_entrypoint_info(RGWSI_Bucket_EP_Ctx& ctx, + const std::string& key, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int read_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const std::string& key, + RGWBucketInfo *info, + real_time *pmtime, + std::map<std::string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp, + rgw_cache_entry_info *cache_info = nullptr, + boost::optional<obj_version> refresh_version = boost::none) override; + + int read_bucket_info(RGWSI_Bucket_X_Ctx& ep_ctx, + const rgw_bucket& bucket, + RGWBucketInfo *info, + real_time *pmtime, + std::map<std::string, bufferlist> *pattrs, + boost::optional<obj_version> refresh_version, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const std::string& key, + RGWBucketInfo& info, + std::optional<RGWBucketInfo *> orig_info, /* nullopt: orig_info was not fetched, + nullptr: orig_info was not found (new bucket instance */ + bool exclusive, + real_time mtime, + std::map<std::string, bufferlist> *pattrs, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int remove_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const std::string& key, + const RGWBucketInfo& bucket_info, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int read_bucket_stats(RGWSI_Bucket_X_Ctx& ctx, + const rgw_bucket& bucket, + RGWBucketEnt *ent, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int read_buckets_stats(RGWSI_Bucket_X_Ctx& ctx, + std::map<std::string, RGWBucketEnt>& m, + optional_yield y, + const DoutPrefixProvider *dpp) override; +}; + diff --git a/src/rgw/services/svc_bucket_sync.h b/src/rgw/services/svc_bucket_sync.h new file mode 100644 index 000000000..7975e062b --- /dev/null +++ b/src/rgw/services/svc_bucket_sync.h @@ -0,0 +1,55 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_bucket_types.h" + +class RGWBucketSyncPolicyHandler; +using RGWBucketSyncPolicyHandlerRef = std::shared_ptr<RGWBucketSyncPolicyHandler>; + + +class RGWSI_Bucket_Sync : public RGWServiceInstance +{ +public: + RGWSI_Bucket_Sync(CephContext *cct) : RGWServiceInstance(cct) {} + + virtual int get_policy_handler(RGWSI_Bucket_X_Ctx& ctx, + std::optional<rgw_zone_id> zone, + std::optional<rgw_bucket> bucket, + RGWBucketSyncPolicyHandlerRef *handler, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int handle_bi_update(const DoutPrefixProvider *dpp, + RGWBucketInfo& bucket_info, + RGWBucketInfo *orig_bucket_info, + optional_yield y) = 0; + virtual int handle_bi_removal(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + optional_yield y) = 0; + + virtual int get_bucket_sync_hints(const DoutPrefixProvider *dpp, + const rgw_bucket& bucket, + std::set<rgw_bucket> *sources, + std::set<rgw_bucket> *dests, + optional_yield y) = 0; +}; + + diff --git a/src/rgw/services/svc_bucket_sync_sobj.cc b/src/rgw/services/svc_bucket_sync_sobj.cc new file mode 100644 index 000000000..ea3398a3f --- /dev/null +++ b/src/rgw/services/svc_bucket_sync_sobj.cc @@ -0,0 +1,903 @@ +#include "svc_bucket_sync_sobj.h" +#include "svc_zone.h" +#include "svc_sys_obj_cache.h" +#include "svc_bucket_sobj.h" + +#include "rgw_bucket_sync.h" +#include "rgw_zone.h" +#include "rgw_sync_policy.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +static string bucket_sync_sources_oid_prefix = "bucket.sync-source-hints"; +static string bucket_sync_targets_oid_prefix = "bucket.sync-target-hints"; + +class RGWSI_Bucket_Sync_SObj_HintIndexManager { + CephContext *cct; + + struct { + RGWSI_Zone *zone; + RGWSI_SysObj *sysobj; + } svc; + +public: + RGWSI_Bucket_Sync_SObj_HintIndexManager(RGWSI_Zone *_zone_svc, + RGWSI_SysObj *_sysobj_svc) { + svc.zone = _zone_svc; + svc.sysobj = _sysobj_svc; + + cct = svc.zone->ctx(); + } + + rgw_raw_obj get_sources_obj(const rgw_bucket& bucket) const; + rgw_raw_obj get_dests_obj(const rgw_bucket& bucket) const; + + template <typename C1, typename C2> + int update_hints(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + C1& added_dests, + C2& removed_dests, + C1& added_sources, + C2& removed_sources, + optional_yield y); +}; + +RGWSI_Bucket_Sync_SObj::RGWSI_Bucket_Sync_SObj(CephContext *cct) : RGWSI_Bucket_Sync(cct) { +} +RGWSI_Bucket_Sync_SObj::~RGWSI_Bucket_Sync_SObj() { +} + +void RGWSI_Bucket_Sync_SObj::init(RGWSI_Zone *_zone_svc, + RGWSI_SysObj *_sysobj_svc, + RGWSI_SysObj_Cache *_cache_svc, + RGWSI_Bucket_SObj *bucket_sobj_svc) +{ + svc.zone = _zone_svc; + svc.sysobj = _sysobj_svc; + svc.cache = _cache_svc; + svc.bucket_sobj = bucket_sobj_svc; + + hint_index_mgr.reset(new RGWSI_Bucket_Sync_SObj_HintIndexManager(svc.zone, svc.sysobj)); +} + +int RGWSI_Bucket_Sync_SObj::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + sync_policy_cache.reset(new RGWChainedCacheImpl<bucket_sync_policy_cache_entry>); + sync_policy_cache->init(svc.cache); + + return 0; +} + +void RGWSI_Bucket_Sync_SObj::get_hint_entities(RGWSI_Bucket_X_Ctx& ctx, + const std::set<rgw_zone_id>& zones, + const std::set<rgw_bucket>& buckets, + std::set<rgw_sync_bucket_entity> *hint_entities, + optional_yield y, const DoutPrefixProvider *dpp) +{ + vector<rgw_bucket> hint_buckets; + + hint_buckets.reserve(buckets.size()); + + for (auto& b : buckets) { + RGWBucketInfo hint_bucket_info; + int ret = svc.bucket_sobj->read_bucket_info(ctx, b, &hint_bucket_info, + nullptr, nullptr, boost::none, + y, dpp); + if (ret < 0) { + ldpp_dout(dpp, 20) << "could not init bucket info for hint bucket=" << b << " ... skipping" << dendl; + continue; + } + + hint_buckets.emplace_back(std::move(hint_bucket_info.bucket)); + } + + for (auto& zone : zones) { + for (auto& b : hint_buckets) { + hint_entities->insert(rgw_sync_bucket_entity(zone, b)); + } + } +} + +int RGWSI_Bucket_Sync_SObj::resolve_policy_hints(RGWSI_Bucket_X_Ctx& ctx, + rgw_sync_bucket_entity& self_entity, + RGWBucketSyncPolicyHandlerRef& handler, + RGWBucketSyncPolicyHandlerRef& zone_policy_handler, + std::map<optional_zone_bucket, RGWBucketSyncPolicyHandlerRef>& temp_map, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + set<rgw_zone_id> source_zones; + set<rgw_zone_id> target_zones; + + zone_policy_handler->reflect(dpp, nullptr, nullptr, + nullptr, nullptr, + &source_zones, + &target_zones, + false); /* relaxed: also get all zones that we allow to sync to/from */ + + std::set<rgw_sync_bucket_entity> hint_entities; + + get_hint_entities(ctx, source_zones, handler->get_source_hints(), &hint_entities, y, dpp); + get_hint_entities(ctx, target_zones, handler->get_target_hints(), &hint_entities, y, dpp); + + std::set<rgw_sync_bucket_pipe> resolved_sources; + std::set<rgw_sync_bucket_pipe> resolved_dests; + + for (auto& hint_entity : hint_entities) { + if (!hint_entity.zone || + !hint_entity.bucket) { + continue; /* shouldn't really happen */ + } + + auto& zid = *hint_entity.zone; + auto& hint_bucket = *hint_entity.bucket; + + RGWBucketSyncPolicyHandlerRef hint_bucket_handler; + + auto iter = temp_map.find(optional_zone_bucket(zid, hint_bucket)); + if (iter != temp_map.end()) { + hint_bucket_handler = iter->second; + } else { + int r = do_get_policy_handler(ctx, zid, hint_bucket, temp_map, &hint_bucket_handler, y, dpp); + if (r < 0) { + ldpp_dout(dpp, 20) << "could not get bucket sync policy handler for hint bucket=" << hint_bucket << " ... skipping" << dendl; + continue; + } + } + + hint_bucket_handler->get_pipes(&resolved_dests, + &resolved_sources, + self_entity); /* flipping resolved dests and sources as these are + relative to the remote entity */ + } + + handler->set_resolved_hints(std::move(resolved_sources), std::move(resolved_dests)); + + return 0; +} + +int RGWSI_Bucket_Sync_SObj::do_get_policy_handler(RGWSI_Bucket_X_Ctx& ctx, + std::optional<rgw_zone_id> zone, + std::optional<rgw_bucket> _bucket, + std::map<optional_zone_bucket, RGWBucketSyncPolicyHandlerRef>& temp_map, + RGWBucketSyncPolicyHandlerRef *handler, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + if (!_bucket) { + *handler = svc.zone->get_sync_policy_handler(zone); + return 0; + } + + auto bucket = *_bucket; + + if (bucket.bucket_id.empty()) { + RGWBucketEntryPoint ep_info; + int ret = svc.bucket_sobj->read_bucket_entrypoint_info(ctx.ep, + RGWSI_Bucket::get_entrypoint_meta_key(bucket), + &ep_info, + nullptr, /* objv_tracker */ + nullptr, /* mtime */ + nullptr, /* attrs */ + y, + dpp, + nullptr, /* cache_info */ + boost::none /* refresh_version */); + if (ret < 0) { + if (ret != -ENOENT) { + ldout(cct, 0) << "ERROR: svc.bucket->read_bucket_info(bucket=" << bucket << ") returned r=" << ret << dendl; + } + return ret; + } + + bucket = ep_info.bucket; + } + + string zone_key; + string bucket_key; + + if (zone && *zone != svc.zone->zone_id()) { + zone_key = zone->id; + } + + bucket_key = RGWSI_Bucket::get_bi_meta_key(bucket); + + string cache_key("bi/" + zone_key + "/" + bucket_key); + + if (auto e = sync_policy_cache->find(cache_key)) { + *handler = e->handler; + return 0; + } + + bucket_sync_policy_cache_entry e; + rgw_cache_entry_info cache_info; + + RGWBucketInfo bucket_info; + map<string, bufferlist> attrs; + + int r = svc.bucket_sobj->read_bucket_instance_info(ctx.bi, + bucket_key, + &bucket_info, + nullptr, + &attrs, + y, + dpp, + &cache_info); + if (r < 0) { + if (r != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: svc.bucket->read_bucket_instance_info(key=" << bucket_key << ") returned r=" << r << dendl; + } + return r; + } + + auto zone_policy_handler = svc.zone->get_sync_policy_handler(zone); + if (!zone_policy_handler) { + ldpp_dout(dpp, 20) << "ERROR: could not find policy handler for zone=" << zone << dendl; + return -ENOENT; + } + + e.handler.reset(zone_policy_handler->alloc_child(bucket_info, std::move(attrs))); + + r = e.handler->init(dpp, y); + if (r < 0) { + ldpp_dout(dpp, 20) << "ERROR: failed to init bucket sync policy handler: r=" << r << dendl; + return r; + } + + temp_map.emplace(optional_zone_bucket{zone, bucket}, e.handler); + + rgw_sync_bucket_entity self_entity(zone.value_or(svc.zone->zone_id()), bucket); + + r = resolve_policy_hints(ctx, self_entity, + e.handler, + zone_policy_handler, + temp_map, y, dpp); + if (r < 0) { + ldpp_dout(dpp, 20) << "ERROR: failed to resolve policy hints: bucket_key=" << bucket_key << ", r=" << r << dendl; + return r; + } + + if (!sync_policy_cache->put(dpp, svc.cache, cache_key, &e, {&cache_info})) { + ldpp_dout(dpp, 20) << "couldn't put bucket_sync_policy cache entry, might have raced with data changes" << dendl; + } + + *handler = e.handler; + + return 0; +} + +int RGWSI_Bucket_Sync_SObj::get_policy_handler(RGWSI_Bucket_X_Ctx& ctx, + std::optional<rgw_zone_id> zone, + std::optional<rgw_bucket> _bucket, + RGWBucketSyncPolicyHandlerRef *handler, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + std::map<optional_zone_bucket, RGWBucketSyncPolicyHandlerRef> temp_map; + return do_get_policy_handler(ctx, zone, _bucket, temp_map, handler, y, dpp); +} + +static bool diff_sets(std::set<rgw_bucket>& orig_set, + std::set<rgw_bucket>& new_set, + vector<rgw_bucket> *added, + vector<rgw_bucket> *removed) +{ + auto oiter = orig_set.begin(); + auto niter = new_set.begin(); + + while (oiter != orig_set.end() && + niter != new_set.end()) { + if (*oiter == *niter) { + ++oiter; + ++niter; + continue; + } else if (*oiter < *niter) { + removed->push_back(*oiter); + ++oiter; + } else { + added->push_back(*niter); + ++niter; + } + } + for (; oiter != orig_set.end(); ++oiter) { + removed->push_back(*oiter); + } + for (; niter != new_set.end(); ++niter) { + added->push_back(*niter); + } + + return !(removed->empty() && added->empty()); +} + + +class RGWSI_BS_SObj_HintIndexObj +{ + friend class RGWSI_Bucket_Sync_SObj; + + CephContext *cct; + struct { + RGWSI_SysObj *sysobj; + } svc; + + rgw_raw_obj obj; + RGWSysObj sysobj; + + RGWObjVersionTracker ot; + + bool has_data{false}; + +public: + struct bi_entry { + rgw_bucket bucket; + map<rgw_bucket /* info_source */, obj_version> sources; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(bucket, bl); + encode(sources, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(bucket, bl); + decode(sources, bl); + DECODE_FINISH(bl); + } + + bool add(const rgw_bucket& info_source, + const obj_version& info_source_ver) { + auto& ver = sources[info_source]; + + if (ver == info_source_ver) { /* already updated */ + return false; + } + + if (info_source_ver.tag == ver.tag && + info_source_ver.ver < ver.ver) { + return false; + } + + ver = info_source_ver; + + return true; + } + + bool remove(const rgw_bucket& info_source, + const obj_version& info_source_ver) { + auto iter = sources.find(info_source); + if (iter == sources.end()) { + return false; + } + + auto& ver = iter->second; + + if (info_source_ver.tag == ver.tag && + info_source_ver.ver < ver.ver) { + return false; + } + + sources.erase(info_source); + return true; + } + + bool empty() const { + return sources.empty(); + } + }; + + struct single_instance_info { + map<rgw_bucket, bi_entry> entries; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(entries, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(entries, bl); + DECODE_FINISH(bl); + } + + bool add_entry(const rgw_bucket& info_source, + const obj_version& info_source_ver, + const rgw_bucket& bucket) { + auto& entry = entries[bucket]; + + if (!entry.add(info_source, info_source_ver)) { + return false; + } + + entry.bucket = bucket; + + return true; + } + + bool remove_entry(const rgw_bucket& info_source, + const obj_version& info_source_ver, + const rgw_bucket& bucket) { + auto iter = entries.find(bucket); + if (iter == entries.end()) { + return false; + } + + if (!iter->second.remove(info_source, info_source_ver)) { + return false; + } + + if (iter->second.empty()) { + entries.erase(iter); + } + + return true; + } + + void clear() { + entries.clear(); + } + + bool empty() const { + return entries.empty(); + } + + void get_entities(std::set<rgw_bucket> *result) const { + for (auto& iter : entries) { + result->insert(iter.first); + } + } + }; + + struct info_map { + map<rgw_bucket, single_instance_info> instances; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(instances, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(instances, bl); + DECODE_FINISH(bl); + } + + bool empty() const { + return instances.empty(); + } + + void clear() { + instances.clear(); + } + + void get_entities(const rgw_bucket& bucket, + std::set<rgw_bucket> *result) const { + auto iter = instances.find(bucket); + if (iter == instances.end()) { + return; + } + iter->second.get_entities(result); + } + } info; + + RGWSI_BS_SObj_HintIndexObj(RGWSI_SysObj *_sysobj_svc, + const rgw_raw_obj& _obj) : cct(_sysobj_svc->ctx()), + obj(_obj), + sysobj(_sysobj_svc->get_obj(obj)) + { + svc.sysobj = _sysobj_svc; + } + + template <typename C1, typename C2> + int update(const DoutPrefixProvider *dpp, + const rgw_bucket& entity, + const RGWBucketInfo& info_source, + C1 *add, + C2 *remove, + optional_yield y); + +private: + template <typename C1, typename C2> + void update_entries(const rgw_bucket& info_source, + const obj_version& info_source_ver, + C1 *add, + C2 *remove, + single_instance_info *instance); + + int read(const DoutPrefixProvider *dpp, optional_yield y); + int flush(const DoutPrefixProvider *dpp, optional_yield y); + + void invalidate() { + has_data = false; + info.clear(); + } + + void get_entities(const rgw_bucket& bucket, + std::set<rgw_bucket> *result) const { + info.get_entities(bucket, result); + } +}; +WRITE_CLASS_ENCODER(RGWSI_BS_SObj_HintIndexObj::bi_entry) +WRITE_CLASS_ENCODER(RGWSI_BS_SObj_HintIndexObj::single_instance_info) +WRITE_CLASS_ENCODER(RGWSI_BS_SObj_HintIndexObj::info_map) + +template <typename C1, typename C2> +int RGWSI_BS_SObj_HintIndexObj::update(const DoutPrefixProvider *dpp, + const rgw_bucket& entity, + const RGWBucketInfo& info_source, + C1 *add, + C2 *remove, + optional_yield y) +{ + int r = 0; + + auto& info_source_ver = info_source.objv_tracker.read_version; + +#define MAX_RETRIES 25 + + for (int i = 0; i < MAX_RETRIES; ++i) { + if (!has_data) { + r = read(dpp, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: cannot update hint index: failed to read: r=" << r << dendl; + return r; + } + } + + auto& instance = info.instances[entity]; + + update_entries(info_source.bucket, + info_source_ver, + add, remove, + &instance); + + if (instance.empty()) { + info.instances.erase(entity); + } + + r = flush(dpp, y); + if (r >= 0) { + return 0; + } + + if (r != -ECANCELED) { + ldpp_dout(dpp, 0) << "ERROR: failed to flush hint index: obj=" << obj << " r=" << r << dendl; + return r; + } + + invalidate(); + } + ldpp_dout(dpp, 0) << "ERROR: failed to flush hint index: too many retries (obj=" << obj << "), likely a bug" << dendl; + + return -EIO; +} + +template <typename C1, typename C2> +void RGWSI_BS_SObj_HintIndexObj::update_entries(const rgw_bucket& info_source, + const obj_version& info_source_ver, + C1 *add, + C2 *remove, + single_instance_info *instance) +{ + if (remove) { + for (auto& bucket : *remove) { + instance->remove_entry(info_source, info_source_ver, bucket); + } + } + + if (add) { + for (auto& bucket : *add) { + instance->add_entry(info_source, info_source_ver, bucket); + } + } +} + +int RGWSI_BS_SObj_HintIndexObj::read(const DoutPrefixProvider *dpp, optional_yield y) { + RGWObjVersionTracker _ot; + bufferlist bl; + int r = sysobj.rop() + .set_objv_tracker(&_ot) /* forcing read of current version */ + .read(dpp, &bl, y); + if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: failed reading data (obj=" << obj << "), r=" << r << dendl; + return r; + } + + ot = _ot; + + if (r >= 0) { + auto iter = bl.cbegin(); + try { + decode(info, iter); + has_data = true; + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to decode entries, ignoring" << dendl; + info.clear(); + } + } else { + info.clear(); + } + + return 0; +} + +int RGWSI_BS_SObj_HintIndexObj::flush(const DoutPrefixProvider *dpp, optional_yield y) { + int r; + + if (!info.empty()) { + bufferlist bl; + encode(info, bl); + + r = sysobj.wop() + .set_objv_tracker(&ot) /* forcing read of current version */ + .write(dpp, bl, y); + + } else { /* remove */ + r = sysobj.wop() + .set_objv_tracker(&ot) + .remove(dpp, y); + } + + if (r < 0) { + return r; + } + + return 0; +} + +rgw_raw_obj RGWSI_Bucket_Sync_SObj_HintIndexManager::get_sources_obj(const rgw_bucket& bucket) const +{ + rgw_bucket b = bucket; + b.bucket_id.clear(); + return rgw_raw_obj(svc.zone->get_zone_params().log_pool, + bucket_sync_sources_oid_prefix + "." + b.get_key()); +} + +rgw_raw_obj RGWSI_Bucket_Sync_SObj_HintIndexManager::get_dests_obj(const rgw_bucket& bucket) const +{ + rgw_bucket b = bucket; + b.bucket_id.clear(); + return rgw_raw_obj(svc.zone->get_zone_params().log_pool, + bucket_sync_targets_oid_prefix + "." + b.get_key()); +} + +template <typename C1, typename C2> +int RGWSI_Bucket_Sync_SObj_HintIndexManager::update_hints(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + C1& added_dests, + C2& removed_dests, + C1& added_sources, + C2& removed_sources, + optional_yield y) +{ + C1 self_entity = { bucket_info.bucket }; + + if (!added_dests.empty() || + !removed_dests.empty()) { + /* update our dests */ + RGWSI_BS_SObj_HintIndexObj index(svc.sysobj, + get_dests_obj(bucket_info.bucket)); + int r = index.update(dpp, bucket_info.bucket, + bucket_info, + &added_dests, + &removed_dests, + y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update targets index for bucket=" << bucket_info.bucket << " r=" << r << dendl; + return r; + } + + /* update dest buckets */ + for (auto& dest_bucket : added_dests) { + RGWSI_BS_SObj_HintIndexObj dep_index(svc.sysobj, + get_sources_obj(dest_bucket)); + int r = dep_index.update(dpp, dest_bucket, + bucket_info, + &self_entity, + static_cast<C2 *>(nullptr), + y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update targets index for bucket=" << dest_bucket << " r=" << r << dendl; + return r; + } + } + /* update removed dest buckets */ + for (auto& dest_bucket : removed_dests) { + RGWSI_BS_SObj_HintIndexObj dep_index(svc.sysobj, + get_sources_obj(dest_bucket)); + int r = dep_index.update(dpp, dest_bucket, + bucket_info, + static_cast<C1 *>(nullptr), + &self_entity, + y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update targets index for bucket=" << dest_bucket << " r=" << r << dendl; + return r; + } + } + } + + if (!added_sources.empty() || + !removed_sources.empty()) { + RGWSI_BS_SObj_HintIndexObj index(svc.sysobj, + get_sources_obj(bucket_info.bucket)); + /* update our sources */ + int r = index.update(dpp, bucket_info.bucket, + bucket_info, + &added_sources, + &removed_sources, + y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update targets index for bucket=" << bucket_info.bucket << " r=" << r << dendl; + return r; + } + + /* update added sources buckets */ + for (auto& source_bucket : added_sources) { + RGWSI_BS_SObj_HintIndexObj dep_index(svc.sysobj, + get_dests_obj(source_bucket)); + int r = dep_index.update(dpp, source_bucket, + bucket_info, + &self_entity, + static_cast<C2 *>(nullptr), + y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update targets index for bucket=" << source_bucket << " r=" << r << dendl; + return r; + } + } + /* update removed dest buckets */ + for (auto& source_bucket : removed_sources) { + RGWSI_BS_SObj_HintIndexObj dep_index(svc.sysobj, + get_dests_obj(source_bucket)); + int r = dep_index.update(dpp, source_bucket, + bucket_info, + static_cast<C1 *>(nullptr), + &self_entity, + y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update targets index for bucket=" << source_bucket << " r=" << r << dendl; + return r; + } + } + } + + return 0; +} + +int RGWSI_Bucket_Sync_SObj::handle_bi_removal(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + optional_yield y) +{ + std::set<rgw_bucket> sources_set; + std::set<rgw_bucket> dests_set; + + if (bucket_info.sync_policy) { + bucket_info.sync_policy->get_potential_related_buckets(bucket_info.bucket, + &sources_set, + &dests_set); + } + + std::vector<rgw_bucket> removed_sources; + removed_sources.reserve(sources_set.size()); + for (auto& e : sources_set) { + removed_sources.push_back(e); + } + + std::vector<rgw_bucket> removed_dests; + removed_dests.reserve(dests_set.size()); + for (auto& e : dests_set) { + removed_dests.push_back(e); + } + + std::vector<rgw_bucket> added_sources; + std::vector<rgw_bucket> added_dests; + + return hint_index_mgr->update_hints(dpp, bucket_info, + added_dests, + removed_dests, + added_sources, + removed_sources, + y); +} + +int RGWSI_Bucket_Sync_SObj::handle_bi_update(const DoutPrefixProvider *dpp, + RGWBucketInfo& bucket_info, + RGWBucketInfo *orig_bucket_info, + optional_yield y) +{ + std::set<rgw_bucket> orig_sources; + std::set<rgw_bucket> orig_dests; + + if (orig_bucket_info && + orig_bucket_info->sync_policy) { + orig_bucket_info->sync_policy->get_potential_related_buckets(bucket_info.bucket, + &orig_sources, + &orig_dests); + } + + std::set<rgw_bucket> sources; + std::set<rgw_bucket> dests; + if (bucket_info.sync_policy) { + bucket_info.sync_policy->get_potential_related_buckets(bucket_info.bucket, + &sources, + &dests); + } + + std::vector<rgw_bucket> removed_sources; + std::vector<rgw_bucket> added_sources; + bool found = diff_sets(orig_sources, sources, &added_sources, &removed_sources); + ldpp_dout(dpp, 20) << __func__ << "(): bucket=" << bucket_info.bucket << ": orig_sources=" << orig_sources << " new_sources=" << sources << dendl; + ldpp_dout(dpp, 20) << __func__ << "(): bucket=" << bucket_info.bucket << ": potential sources added=" << added_sources << " removed=" << removed_sources << dendl; + + std::vector<rgw_bucket> removed_dests; + std::vector<rgw_bucket> added_dests; + found = found || diff_sets(orig_dests, dests, &added_dests, &removed_dests); + + ldpp_dout(dpp, 20) << __func__ << "(): bucket=" << bucket_info.bucket << ": orig_dests=" << orig_dests << " new_dests=" << dests << dendl; + ldpp_dout(dpp, 20) << __func__ << "(): bucket=" << bucket_info.bucket << ": potential dests added=" << added_dests << " removed=" << removed_dests << dendl; + + if (!found) { + return 0; + } + + return hint_index_mgr->update_hints(dpp, bucket_info, + dests, /* set all dests, not just the ones that were added */ + removed_dests, + sources, /* set all sources, not just that the ones that were added */ + removed_sources, + y); +} + +int RGWSI_Bucket_Sync_SObj::get_bucket_sync_hints(const DoutPrefixProvider *dpp, + const rgw_bucket& bucket, + std::set<rgw_bucket> *sources, + std::set<rgw_bucket> *dests, + optional_yield y) +{ + if (!sources && !dests) { + return 0; + } + + if (sources) { + RGWSI_BS_SObj_HintIndexObj index(svc.sysobj, + hint_index_mgr->get_sources_obj(bucket)); + int r = index.read(dpp, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to update sources index for bucket=" << bucket << " r=" << r << dendl; + return r; + } + + index.get_entities(bucket, sources); + + if (!bucket.bucket_id.empty()) { + rgw_bucket b = bucket; + b.bucket_id.clear(); + index.get_entities(b, sources); + } + } + + if (dests) { + RGWSI_BS_SObj_HintIndexObj index(svc.sysobj, + hint_index_mgr->get_dests_obj(bucket)); + int r = index.read(dpp, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to read targets index for bucket=" << bucket << " r=" << r << dendl; + return r; + } + + index.get_entities(bucket, dests); + + if (!bucket.bucket_id.empty()) { + rgw_bucket b = bucket; + b.bucket_id.clear(); + index.get_entities(b, dests); + } + } + + return 0; +} diff --git a/src/rgw/services/svc_bucket_sync_sobj.h b/src/rgw/services/svc_bucket_sync_sobj.h new file mode 100644 index 000000000..779df7b99 --- /dev/null +++ b/src/rgw/services/svc_bucket_sync_sobj.h @@ -0,0 +1,123 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_meta_be.h" +#include "svc_bucket_sync.h" + +class RGWSI_Zone; +class RGWSI_SysObj_Cache; +class RGWSI_Bucket_SObj; + +template <class T> +class RGWChainedCacheImpl; + +class RGWSI_Bucket_Sync_SObj_HintIndexManager; + +struct rgw_sync_bucket_entity; + +class RGWSI_Bucket_Sync_SObj : public RGWSI_Bucket_Sync +{ + struct bucket_sync_policy_cache_entry { + std::shared_ptr<RGWBucketSyncPolicyHandler> handler; + }; + + std::unique_ptr<RGWChainedCacheImpl<bucket_sync_policy_cache_entry> > sync_policy_cache; + + std::unique_ptr<RGWSI_Bucket_Sync_SObj_HintIndexManager> hint_index_mgr; + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + + struct optional_zone_bucket { + std::optional<rgw_zone_id> zone; + std::optional<rgw_bucket> bucket; + + optional_zone_bucket(const std::optional<rgw_zone_id>& _zone, + const std::optional<rgw_bucket>& _bucket) : zone(_zone), bucket(_bucket) {} + + bool operator<(const optional_zone_bucket& ozb) const { + if (zone < ozb.zone) { + return true; + } + if (zone > ozb.zone) { + return false; + } + return bucket < ozb.bucket; + } + }; + + void get_hint_entities(RGWSI_Bucket_X_Ctx& ctx, + const std::set<rgw_zone_id>& zone_names, + const std::set<rgw_bucket>& buckets, + std::set<rgw_sync_bucket_entity> *hint_entities, + optional_yield y, const DoutPrefixProvider *); + int resolve_policy_hints(RGWSI_Bucket_X_Ctx& ctx, + rgw_sync_bucket_entity& self_entity, + RGWBucketSyncPolicyHandlerRef& handler, + RGWBucketSyncPolicyHandlerRef& zone_policy_handler, + std::map<optional_zone_bucket, RGWBucketSyncPolicyHandlerRef>& temp_map, + optional_yield y, + const DoutPrefixProvider *dpp); + int do_get_policy_handler(RGWSI_Bucket_X_Ctx& ctx, + std::optional<rgw_zone_id> zone, + std::optional<rgw_bucket> _bucket, + std::map<optional_zone_bucket, RGWBucketSyncPolicyHandlerRef>& temp_map, + RGWBucketSyncPolicyHandlerRef *handler, + optional_yield y, + const DoutPrefixProvider *dpp); +public: + struct Svc { + RGWSI_Zone *zone{nullptr}; + RGWSI_SysObj *sysobj{nullptr}; + RGWSI_SysObj_Cache *cache{nullptr}; + RGWSI_Bucket_SObj *bucket_sobj{nullptr}; + } svc; + + RGWSI_Bucket_Sync_SObj(CephContext *cct); + ~RGWSI_Bucket_Sync_SObj(); + + void init(RGWSI_Zone *_zone_svc, + RGWSI_SysObj *_sysobj_svc, + RGWSI_SysObj_Cache *_cache_svc, + RGWSI_Bucket_SObj *_bucket_sobj_svc); + + + int get_policy_handler(RGWSI_Bucket_X_Ctx& ctx, + std::optional<rgw_zone_id> zone, + std::optional<rgw_bucket> bucket, + RGWBucketSyncPolicyHandlerRef *handler, + optional_yield y, + const DoutPrefixProvider *dpp); + + int handle_bi_update(const DoutPrefixProvider *dpp, + RGWBucketInfo& bucket_info, + RGWBucketInfo *orig_bucket_info, + optional_yield y) override; + int handle_bi_removal(const DoutPrefixProvider *dpp, + const RGWBucketInfo& bucket_info, + optional_yield y) override; + + int get_bucket_sync_hints(const DoutPrefixProvider *dpp, + const rgw_bucket& bucket, + std::set<rgw_bucket> *sources, + std::set<rgw_bucket> *dests, + optional_yield y) override; +}; + diff --git a/src/rgw/services/svc_bucket_types.h b/src/rgw/services/svc_bucket_types.h new file mode 100644 index 000000000..30e5309d5 --- /dev/null +++ b/src/rgw/services/svc_bucket_types.h @@ -0,0 +1,38 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "common/ptr_wrapper.h" + +#include "svc_meta_be.h" +#include "svc_meta_be_types.h" + +class RGWSI_MetaBackend_Handler; + +using RGWSI_Bucket_BE_Handler = ptr_wrapper<RGWSI_MetaBackend_Handler, RGWSI_META_BE_TYPES::BUCKET>; +using RGWSI_BucketInstance_BE_Handler = ptr_wrapper<RGWSI_MetaBackend_Handler, RGWSI_META_BE_TYPES::BI>; + + +using RGWSI_Bucket_EP_Ctx = ptr_wrapper<RGWSI_MetaBackend::Context, RGWSI_META_BE_TYPES::BUCKET>; +using RGWSI_Bucket_BI_Ctx = ptr_wrapper<RGWSI_MetaBackend::Context, RGWSI_META_BE_TYPES::BI>; + +struct RGWSI_Bucket_X_Ctx { + RGWSI_Bucket_EP_Ctx ep; + RGWSI_Bucket_BI_Ctx bi; +}; + diff --git a/src/rgw/services/svc_cls.cc b/src/rgw/services/svc_cls.cc new file mode 100644 index 000000000..342146bfe --- /dev/null +++ b/src/rgw/services/svc_cls.cc @@ -0,0 +1,478 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + + +#include "svc_cls.h" +#include "svc_rados.h" +#include "svc_zone.h" + +#include "rgw_zone.h" + +#include "cls/otp/cls_otp_client.h" +#include "cls/log/cls_log_client.h" +#include "cls/lock/cls_lock_client.h" + + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +static string log_lock_name = "rgw_log_lock"; + +int RGWSI_Cls::do_start(optional_yield y, const DoutPrefixProvider *dpp) +{ + int r = mfa.do_start(y, dpp); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to start mfa service" << dendl; + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::get_mfa_obj(const DoutPrefixProvider *dpp, const rgw_user& user, std::optional<RGWSI_RADOS::Obj> *obj) +{ + string oid = get_mfa_oid(user); + rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid); + + obj->emplace(rados_svc->obj(o)); + int r = (*obj)->open(dpp); + if (r < 0) { + ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl; + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::get_mfa_ref(const DoutPrefixProvider *dpp, const rgw_user& user, rgw_rados_ref *ref) +{ + std::optional<RGWSI_RADOS::Obj> obj; + int r = get_mfa_obj(dpp, user, &obj); + if (r < 0) { + return r; + } + *ref = obj->get_ref(); + return 0; +} + +int RGWSI_Cls::MFA::check_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, const string& otp_id, const string& pin, optional_yield y) +{ + rgw_rados_ref ref; + int r = get_mfa_ref(dpp, user, &ref); + if (r < 0) { + return r; + } + + rados::cls::otp::otp_check_t result; + + r = rados::cls::otp::OTP::check(cct, ref.pool.ioctx(), ref.obj.oid, otp_id, pin, &result); + if (r < 0) + return r; + + ldpp_dout(dpp, 20) << "OTP check, otp_id=" << otp_id << " result=" << (int)result.result << dendl; + + return (result.result == rados::cls::otp::OTP_CHECK_SUCCESS ? 0 : -EACCES); +} + +void RGWSI_Cls::MFA::prepare_mfa_write(librados::ObjectWriteOperation *op, + RGWObjVersionTracker *objv_tracker, + const ceph::real_time& mtime) +{ + RGWObjVersionTracker ot; + + if (objv_tracker) { + ot = *objv_tracker; + } + + if (ot.write_version.tag.empty()) { + if (ot.read_version.tag.empty()) { + ot.generate_new_write_ver(cct); + } else { + ot.write_version = ot.read_version; + ot.write_version.ver++; + } + } + + ot.prepare_op_for_write(op); + struct timespec mtime_ts = real_clock::to_timespec(mtime); + op->mtime2(&mtime_ts); +} + +int RGWSI_Cls::MFA::create_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, const rados::cls::otp::otp_info_t& config, + RGWObjVersionTracker *objv_tracker, const ceph::real_time& mtime, optional_yield y) +{ + std::optional<RGWSI_RADOS::Obj> obj; + int r = get_mfa_obj(dpp, user, &obj); + if (r < 0) { + return r; + } + + librados::ObjectWriteOperation op; + prepare_mfa_write(&op, objv_tracker, mtime); + rados::cls::otp::OTP::create(&op, config); + r = obj->operate(dpp, &op, y); + if (r < 0) { + ldpp_dout(dpp, 20) << "OTP create, otp_id=" << config.id << " result=" << (int)r << dendl; + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::remove_mfa(const DoutPrefixProvider *dpp, + const rgw_user& user, const string& id, + RGWObjVersionTracker *objv_tracker, + const ceph::real_time& mtime, + optional_yield y) +{ + std::optional<RGWSI_RADOS::Obj> obj; + int r = get_mfa_obj(dpp, user, &obj); + if (r < 0) { + return r; + } + + librados::ObjectWriteOperation op; + prepare_mfa_write(&op, objv_tracker, mtime); + rados::cls::otp::OTP::remove(&op, id); + r = obj->operate(dpp, &op, y); + if (r < 0) { + ldpp_dout(dpp, 20) << "OTP remove, otp_id=" << id << " result=" << (int)r << dendl; + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::get_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, const string& id, rados::cls::otp::otp_info_t *result, + optional_yield y) +{ + rgw_rados_ref ref; + + int r = get_mfa_ref(dpp, user, &ref); + if (r < 0) { + return r; + } + + r = rados::cls::otp::OTP::get(nullptr, ref.pool.ioctx(), ref.obj.oid, id, result); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::list_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, list<rados::cls::otp::otp_info_t> *result, + optional_yield y) +{ + rgw_rados_ref ref; + + int r = get_mfa_ref(dpp, user, &ref); + if (r < 0) { + return r; + } + + r = rados::cls::otp::OTP::get_all(nullptr, ref.pool.ioctx(), ref.obj.oid, result); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::otp_get_current_time(const DoutPrefixProvider *dpp, const rgw_user& user, ceph::real_time *result, + optional_yield y) +{ + rgw_rados_ref ref; + + int r = get_mfa_ref(dpp, user, &ref); + if (r < 0) { + return r; + } + + r = rados::cls::otp::OTP::get_current_time(ref.pool.ioctx(), ref.obj.oid, result); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::set_mfa(const DoutPrefixProvider *dpp, const string& oid, const list<rados::cls::otp::otp_info_t>& entries, + bool reset_obj, RGWObjVersionTracker *objv_tracker, + const real_time& mtime, + optional_yield y) +{ + rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid); + auto obj = rados_svc->obj(o); + int r = obj.open(dpp); + if (r < 0) { + ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl; + return r; + } + librados::ObjectWriteOperation op; + if (reset_obj) { + op.remove(); + op.set_op_flags2(LIBRADOS_OP_FLAG_FAILOK); + op.create(false); + } + prepare_mfa_write(&op, objv_tracker, mtime); + rados::cls::otp::OTP::set(&op, entries); + r = obj.operate(dpp, &op, y); + if (r < 0) { + ldpp_dout(dpp, 20) << "OTP set entries.size()=" << entries.size() << " result=" << (int)r << dendl; + return r; + } + + return 0; +} + +int RGWSI_Cls::MFA::list_mfa(const DoutPrefixProvider *dpp, const string& oid, list<rados::cls::otp::otp_info_t> *result, + RGWObjVersionTracker *objv_tracker, ceph::real_time *pmtime, + optional_yield y) +{ + rgw_raw_obj o(zone_svc->get_zone_params().otp_pool, oid); + auto obj = rados_svc->obj(o); + int r = obj.open(dpp); + if (r < 0) { + ldpp_dout(dpp, 4) << "failed to open rados context for " << o << dendl; + return r; + } + auto& ref = obj.get_ref(); + librados::ObjectReadOperation op; + struct timespec mtime_ts; + if (pmtime) { + op.stat2(nullptr, &mtime_ts, nullptr); + } + objv_tracker->prepare_op_for_read(&op); + r = rados::cls::otp::OTP::get_all(&op, ref.pool.ioctx(), ref.obj.oid, result); + if (r < 0) { + return r; + } + if (pmtime) { + *pmtime = ceph::real_clock::from_timespec(mtime_ts); + } + + return 0; +} + +void RGWSI_Cls::TimeLog::prepare_entry(cls_log_entry& entry, + const real_time& ut, + const string& section, + const string& key, + bufferlist& bl) +{ + cls_log_add_prepare_entry(entry, utime_t(ut), section, key, bl); +} + +int RGWSI_Cls::TimeLog::init_obj(const DoutPrefixProvider *dpp, const string& oid, RGWSI_RADOS::Obj& obj) +{ + rgw_raw_obj o(zone_svc->get_zone_params().log_pool, oid); + obj = rados_svc->obj(o); + return obj.open(dpp); + +} +int RGWSI_Cls::TimeLog::add(const DoutPrefixProvider *dpp, + const string& oid, + const real_time& ut, + const string& section, + const string& key, + bufferlist& bl, + optional_yield y) +{ + RGWSI_RADOS::Obj obj; + + int r = init_obj(dpp, oid, obj); + if (r < 0) { + return r; + } + + librados::ObjectWriteOperation op; + utime_t t(ut); + cls_log_add(op, t, section, key, bl); + + return obj.operate(dpp, &op, y); +} + +int RGWSI_Cls::TimeLog::add(const DoutPrefixProvider *dpp, + const string& oid, + std::list<cls_log_entry>& entries, + librados::AioCompletion *completion, + bool monotonic_inc, + optional_yield y) +{ + RGWSI_RADOS::Obj obj; + + int r = init_obj(dpp, oid, obj); + if (r < 0) { + return r; + } + + librados::ObjectWriteOperation op; + cls_log_add(op, entries, monotonic_inc); + + if (!completion) { + r = obj.operate(dpp, &op, y); + } else { + r = obj.aio_operate(completion, &op); + } + return r; +} + +int RGWSI_Cls::TimeLog::list(const DoutPrefixProvider *dpp, + const string& oid, + const real_time& start_time, + const real_time& end_time, + int max_entries, std::list<cls_log_entry>& entries, + const string& marker, + string *out_marker, + bool *truncated, + optional_yield y) +{ + RGWSI_RADOS::Obj obj; + + int r = init_obj(dpp, oid, obj); + if (r < 0) { + return r; + } + + librados::ObjectReadOperation op; + + utime_t st(start_time); + utime_t et(end_time); + + cls_log_list(op, st, et, marker, max_entries, entries, + out_marker, truncated); + + bufferlist obl; + + int ret = obj.operate(dpp, &op, &obl, y); + if (ret < 0) + return ret; + + return 0; +} + +int RGWSI_Cls::TimeLog::info(const DoutPrefixProvider *dpp, + const string& oid, + cls_log_header *header, + optional_yield y) +{ + RGWSI_RADOS::Obj obj; + + int r = init_obj(dpp, oid, obj); + if (r < 0) { + return r; + } + + librados::ObjectReadOperation op; + + cls_log_info(op, header); + + bufferlist obl; + + int ret = obj.operate(dpp, &op, &obl, y); + if (ret < 0) + return ret; + + return 0; +} + +int RGWSI_Cls::TimeLog::info_async(const DoutPrefixProvider *dpp, + RGWSI_RADOS::Obj& obj, + const string& oid, + cls_log_header *header, + librados::AioCompletion *completion) +{ + int r = init_obj(dpp, oid, obj); + if (r < 0) { + return r; + } + + librados::ObjectReadOperation op; + + cls_log_info(op, header); + + int ret = obj.aio_operate(completion, &op, nullptr); + if (ret < 0) + return ret; + + return 0; +} + +int RGWSI_Cls::TimeLog::trim(const DoutPrefixProvider *dpp, + const string& oid, + const real_time& start_time, + const real_time& end_time, + const string& from_marker, + const string& to_marker, + librados::AioCompletion *completion, + optional_yield y) +{ + RGWSI_RADOS::Obj obj; + + int r = init_obj(dpp, oid, obj); + if (r < 0) { + return r; + } + + utime_t st(start_time); + utime_t et(end_time); + + librados::ObjectWriteOperation op; + cls_log_trim(op, st, et, from_marker, to_marker); + + if (!completion) { + r = obj.operate(dpp, &op, y); + } else { + r = obj.aio_operate(completion, &op); + } + return r; +} + +int RGWSI_Cls::Lock::lock_exclusive(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + const string& oid, + timespan& duration, + string& zone_id, + string& owner_id, + std::optional<string> lock_name) +{ + auto p = rados_svc->pool(pool); + int r = p.open(dpp); + if (r < 0) { + return r; + } + + uint64_t msec = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count(); + utime_t ut(msec / 1000, msec % 1000); + + rados::cls::lock::Lock l(lock_name.value_or(log_lock_name)); + l.set_duration(ut); + l.set_cookie(owner_id); + l.set_tag(zone_id); + l.set_may_renew(true); + + return l.lock_exclusive(&p.ioctx(), oid); +} + +int RGWSI_Cls::Lock::unlock(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + const string& oid, + string& zone_id, + string& owner_id, + std::optional<string> lock_name) +{ + auto p = rados_svc->pool(pool); + int r = p.open(dpp); + if (r < 0) { + return r; + } + + rados::cls::lock::Lock l(lock_name.value_or(log_lock_name)); + l.set_tag(zone_id); + l.set_cookie(owner_id); + + return l.unlock(&p.ioctx(), oid); +} + diff --git a/src/rgw/services/svc_cls.h b/src/rgw/services/svc_cls.h new file mode 100644 index 000000000..d1d1d659b --- /dev/null +++ b/src/rgw/services/svc_cls.h @@ -0,0 +1,166 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "cls/otp/cls_otp_types.h" +#include "cls/log/cls_log_types.h" + +#include "rgw_service.h" + +#include "svc_rados.h" + + +class RGWSI_Cls : public RGWServiceInstance +{ + RGWSI_Zone *zone_svc{nullptr}; + RGWSI_RADOS *rados_svc{nullptr}; + + class ClsSubService : public RGWServiceInstance { + friend class RGWSI_Cls; + + RGWSI_Cls *cls_svc{nullptr}; + RGWSI_Zone *zone_svc{nullptr}; + RGWSI_RADOS *rados_svc{nullptr}; + + void init(RGWSI_Cls *_cls_svc, RGWSI_Zone *_zone_svc, RGWSI_RADOS *_rados_svc) { + cls_svc = _cls_svc; + zone_svc = _cls_svc->zone_svc; + rados_svc = _cls_svc->rados_svc; + } + + public: + ClsSubService(CephContext *cct) : RGWServiceInstance(cct) {} + }; + +public: + class MFA : public ClsSubService { + int get_mfa_obj(const DoutPrefixProvider *dpp, const rgw_user& user, std::optional<RGWSI_RADOS::Obj> *obj); + int get_mfa_ref(const DoutPrefixProvider *dpp, const rgw_user& user, rgw_rados_ref *ref); + + void prepare_mfa_write(librados::ObjectWriteOperation *op, + RGWObjVersionTracker *objv_tracker, + const ceph::real_time& mtime); + + public: + MFA(CephContext *cct): ClsSubService(cct) {} + + std::string get_mfa_oid(const rgw_user& user) { + return std::string("user:") + user.to_str(); + } + + int check_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, const std::string& otp_id, const std::string& pin, optional_yield y); + int create_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, const rados::cls::otp::otp_info_t& config, + RGWObjVersionTracker *objv_tracker, const ceph::real_time& mtime, optional_yield y); + int remove_mfa(const DoutPrefixProvider *dpp, + const rgw_user& user, const std::string& id, + RGWObjVersionTracker *objv_tracker, + const ceph::real_time& mtime, + optional_yield y); + int get_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, const std::string& id, rados::cls::otp::otp_info_t *result, optional_yield y); + int list_mfa(const DoutPrefixProvider *dpp, const rgw_user& user, std::list<rados::cls::otp::otp_info_t> *result, optional_yield y); + int otp_get_current_time(const DoutPrefixProvider *dpp, const rgw_user& user, ceph::real_time *result, optional_yield y); + int set_mfa(const DoutPrefixProvider *dpp, const std::string& oid, const std::list<rados::cls::otp::otp_info_t>& entries, + bool reset_obj, RGWObjVersionTracker *objv_tracker, + const real_time& mtime, optional_yield y); + int list_mfa(const DoutPrefixProvider *dpp, const std::string& oid, std::list<rados::cls::otp::otp_info_t> *result, + RGWObjVersionTracker *objv_tracker, ceph::real_time *pmtime, optional_yield y); + } mfa; + + class TimeLog : public ClsSubService { + int init_obj(const DoutPrefixProvider *dpp, const std::string& oid, RGWSI_RADOS::Obj& obj); + public: + TimeLog(CephContext *cct): ClsSubService(cct) {} + + void prepare_entry(cls_log_entry& entry, + const real_time& ut, + const std::string& section, + const std::string& key, + bufferlist& bl); + int add(const DoutPrefixProvider *dpp, + const std::string& oid, + const real_time& ut, + const std::string& section, + const std::string& key, + bufferlist& bl, + optional_yield y); + int add(const DoutPrefixProvider *dpp, + const std::string& oid, + std::list<cls_log_entry>& entries, + librados::AioCompletion *completion, + bool monotonic_inc, + optional_yield y); + int list(const DoutPrefixProvider *dpp, + const std::string& oid, + const real_time& start_time, + const real_time& end_time, + int max_entries, std::list<cls_log_entry>& entries, + const std::string& marker, + std::string *out_marker, + bool *truncated, + optional_yield y); + int info(const DoutPrefixProvider *dpp, + const std::string& oid, + cls_log_header *header, + optional_yield y); + int info_async(const DoutPrefixProvider *dpp, + RGWSI_RADOS::Obj& obj, + const std::string& oid, + cls_log_header *header, + librados::AioCompletion *completion); + int trim(const DoutPrefixProvider *dpp, + const std::string& oid, + const real_time& start_time, + const real_time& end_time, + const std::string& from_marker, + const std::string& to_marker, + librados::AioCompletion *completion, + optional_yield y); + } timelog; + + class Lock : public ClsSubService { + int init_obj(const std::string& oid, RGWSI_RADOS::Obj& obj); + public: + Lock(CephContext *cct): ClsSubService(cct) {} + int lock_exclusive(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + const std::string& oid, + timespan& duration, + std::string& zone_id, + std::string& owner_id, + std::optional<std::string> lock_name = std::nullopt); + int unlock(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + const std::string& oid, + std::string& zone_id, + std::string& owner_id, + std::optional<std::string> lock_name = std::nullopt); + } lock; + + RGWSI_Cls(CephContext *cct): RGWServiceInstance(cct), mfa(cct), timelog(cct), lock(cct) {} + + void init(RGWSI_Zone *_zone_svc, RGWSI_RADOS *_rados_svc) { + rados_svc = _rados_svc; + zone_svc = _zone_svc; + + mfa.init(this, zone_svc, rados_svc); + timelog.init(this, zone_svc, rados_svc); + lock.init(this, zone_svc, rados_svc); + } + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; +}; + diff --git a/src/rgw/services/svc_config_key.h b/src/rgw/services/svc_config_key.h new file mode 100644 index 000000000..1c068b795 --- /dev/null +++ b/src/rgw/services/svc_config_key.h @@ -0,0 +1,31 @@ + + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +class RGWSI_ConfigKey : public RGWServiceInstance +{ +public: + RGWSI_ConfigKey(CephContext *cct) : RGWServiceInstance(cct) {} + virtual ~RGWSI_ConfigKey() {} + + virtual int get(const std::string& key, bool secure, bufferlist *result) = 0; +}; + diff --git a/src/rgw/services/svc_config_key_rados.cc b/src/rgw/services/svc_config_key_rados.cc new file mode 100644 index 000000000..5edb02ea7 --- /dev/null +++ b/src/rgw/services/svc_config_key_rados.cc @@ -0,0 +1,50 @@ + +#include "svc_rados.h" +#include "svc_config_key_rados.h" + +using namespace std; + +RGWSI_ConfigKey_RADOS::~RGWSI_ConfigKey_RADOS(){} + +int RGWSI_ConfigKey_RADOS::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + maybe_insecure_mon_conn = !svc.rados->check_secure_mon_conn(dpp); + + return 0; +} + +void RGWSI_ConfigKey_RADOS::warn_if_insecure() +{ + if (!maybe_insecure_mon_conn || + warned_insecure.test_and_set()) { + return; + } + + string s = "rgw is configured to optionally allow insecure connections to the monitors (auth_supported, ms_mon_client_mode), ssl certificates stored at the monitor configuration could leak"; + + svc.rados->clog_warn(s); + + lderr(ctx()) << __func__ << "(): WARNING: " << s << dendl; +} + +int RGWSI_ConfigKey_RADOS::get(const string& key, bool secure, bufferlist *result) +{ + string cmd = + "{" + "\"prefix\": \"config-key get\", " + "\"key\": \"" + key + "\"" + "}"; + + bufferlist inbl; + auto handle = svc.rados->handle(); + int ret = handle.mon_command(cmd, inbl, result, nullptr); + if (ret < 0) { + return ret; + } + + if (secure) { + warn_if_insecure(); + } + + return 0; +} diff --git a/src/rgw/services/svc_config_key_rados.h b/src/rgw/services/svc_config_key_rados.h new file mode 100644 index 000000000..b3b995ac7 --- /dev/null +++ b/src/rgw/services/svc_config_key_rados.h @@ -0,0 +1,54 @@ + + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include <atomic> + +#include "rgw_service.h" + +#include "svc_config_key.h" + +class RGWSI_RADOS; + +class RGWSI_ConfigKey_RADOS : public RGWSI_ConfigKey +{ + bool maybe_insecure_mon_conn{false}; + std::atomic_flag warned_insecure = ATOMIC_FLAG_INIT; + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + + void warn_if_insecure(); + +public: + struct Svc { + RGWSI_RADOS *rados{nullptr}; + } svc; + + void init(RGWSI_RADOS *rados_svc) { + svc.rados = rados_svc; + } + + RGWSI_ConfigKey_RADOS(CephContext *cct) : RGWSI_ConfigKey(cct) {} + + virtual ~RGWSI_ConfigKey_RADOS() override; + + int get(const std::string& key, bool secure, bufferlist *result) override; +}; + + diff --git a/src/rgw/services/svc_finisher.cc b/src/rgw/services/svc_finisher.cc new file mode 100644 index 000000000..4883c7c50 --- /dev/null +++ b/src/rgw/services/svc_finisher.cc @@ -0,0 +1,58 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "common/Finisher.h" + +#include "svc_finisher.h" + +using namespace std; + +int RGWSI_Finisher::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + finisher = new Finisher(cct); + finisher->start(); + + return 0; +} + +void RGWSI_Finisher::shutdown() +{ + if (finalized) { + return; + } + + if (finisher) { + finisher->stop(); + + map<int, ShutdownCB *> cbs; + cbs.swap(shutdown_cbs); /* move cbs out, in case caller unregisters */ + for (auto& iter : cbs) { + iter.second->call(); + } + delete finisher; + } + + finalized = true; +} + +RGWSI_Finisher::~RGWSI_Finisher() +{ + shutdown(); +} + +void RGWSI_Finisher::register_caller(ShutdownCB *cb, int *phandle) +{ + *phandle = ++handles_counter; + shutdown_cbs[*phandle] = cb; +} + +void RGWSI_Finisher::unregister_caller(int handle) +{ + shutdown_cbs.erase(handle); +} + +void RGWSI_Finisher::schedule_context(Context *c) +{ + finisher->queue(c); +} + diff --git a/src/rgw/services/svc_finisher.h b/src/rgw/services/svc_finisher.h new file mode 100644 index 000000000..911b48f2b --- /dev/null +++ b/src/rgw/services/svc_finisher.h @@ -0,0 +1,44 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" + +class Context; +class Finisher; + +class RGWSI_Finisher : public RGWServiceInstance +{ + friend struct RGWServices_Def; +public: + class ShutdownCB; + +private: + Finisher *finisher{nullptr}; + bool finalized{false}; + + void shutdown() override; + + std::map<int, ShutdownCB *> shutdown_cbs; + std::atomic<int> handles_counter{0}; + +protected: + void init() {} + int do_start(optional_yield y, const DoutPrefixProvider *dpp) override; + +public: + RGWSI_Finisher(CephContext *cct): RGWServiceInstance(cct) {} + ~RGWSI_Finisher(); + + class ShutdownCB { + public: + virtual ~ShutdownCB() {} + virtual void call() = 0; + }; + + void register_caller(ShutdownCB *cb, int *phandle); + void unregister_caller(int handle); + + void schedule_context(Context *c); +}; diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc new file mode 100644 index 000000000..09a68d3d7 --- /dev/null +++ b/src/rgw/services/svc_mdlog.cc @@ -0,0 +1,549 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_mdlog.h" +#include "svc_rados.h" +#include "svc_zone.h" +#include "svc_sys_obj.h" + +#include "rgw_tools.h" +#include "rgw_mdlog.h" +#include "rgw_coroutine.h" +#include "rgw_cr_rados.h" +#include "rgw_zone.h" + +#include "common/errno.h" + +#include <boost/asio/yield.hpp> + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +using Svc = RGWSI_MDLog::Svc; +using Cursor = RGWPeriodHistory::Cursor; + +RGWSI_MDLog::RGWSI_MDLog(CephContext *cct, bool _run_sync) : RGWServiceInstance(cct), run_sync(_run_sync) { +} + +RGWSI_MDLog::~RGWSI_MDLog() { +} + +int RGWSI_MDLog::init(RGWSI_RADOS *_rados_svc, RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, RGWSI_Cls *_cls_svc) +{ + svc.zone = _zone_svc; + svc.sysobj = _sysobj_svc; + svc.mdlog = this; + svc.rados = _rados_svc; + svc.cls = _cls_svc; + + return 0; +} + +int RGWSI_MDLog::do_start(optional_yield y, const DoutPrefixProvider *dpp) +{ + auto& current_period = svc.zone->get_current_period(); + + current_log = get_log(current_period.get_id()); + + period_puller.reset(new RGWPeriodPuller(svc.zone, svc.sysobj)); + period_history.reset(new RGWPeriodHistory(cct, period_puller.get(), + current_period)); + + if (run_sync && + svc.zone->need_to_sync()) { + // initialize the log period history + svc.mdlog->init_oldest_log_period(y, dpp); + } + return 0; +} + +int RGWSI_MDLog::read_history(RGWMetadataLogHistory *state, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) const +{ + auto& pool = svc.zone->get_zone_params().log_pool; + const auto& oid = RGWMetadataLogHistory::oid; + bufferlist bl; + int ret = rgw_get_system_obj(svc.sysobj, pool, oid, bl, objv_tracker, nullptr, y, dpp); + if (ret < 0) { + return ret; + } + if (bl.length() == 0) { + /* bad history object, remove it */ + rgw_raw_obj obj(pool, oid); + auto sysobj = svc.sysobj->get_obj(obj); + ret = sysobj.wop().remove(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: meta history is empty, but cannot remove it (" << cpp_strerror(-ret) << ")" << dendl; + return ret; + } + return -ENOENT; + } + try { + auto p = bl.cbegin(); + state->decode(p); + } catch (buffer::error& e) { + ldpp_dout(dpp, 1) << "failed to decode the mdlog history: " + << e.what() << dendl; + return -EIO; + } + return 0; +} + +int RGWSI_MDLog::write_history(const DoutPrefixProvider *dpp, + const RGWMetadataLogHistory& state, + RGWObjVersionTracker *objv_tracker, + optional_yield y, bool exclusive) +{ + bufferlist bl; + state.encode(bl); + + auto& pool = svc.zone->get_zone_params().log_pool; + const auto& oid = RGWMetadataLogHistory::oid; + return rgw_put_system_obj(dpp, svc.sysobj, pool, oid, bl, + exclusive, objv_tracker, real_time{}, y); +} + +namespace mdlog { + +using Cursor = RGWPeriodHistory::Cursor; + +namespace { +template <class T> +class SysObjReadCR : public RGWSimpleCoroutine { + const DoutPrefixProvider *dpp; + RGWAsyncRadosProcessor *async_rados; + RGWSI_SysObj *svc; + + rgw_raw_obj obj; + T *result; + /// on ENOENT, call handle_data() with an empty object instead of failing + const bool empty_on_enoent; + RGWObjVersionTracker *objv_tracker; + RGWAsyncGetSystemObj *req{nullptr}; + +public: + SysObjReadCR(const DoutPrefixProvider *_dpp, + RGWAsyncRadosProcessor *_async_rados, RGWSI_SysObj *_svc, + const rgw_raw_obj& _obj, + T *_result, bool empty_on_enoent = true, + RGWObjVersionTracker *objv_tracker = nullptr) + : RGWSimpleCoroutine(_svc->ctx()), dpp(_dpp), async_rados(_async_rados), svc(_svc), + obj(_obj), result(_result), + empty_on_enoent(empty_on_enoent), objv_tracker(objv_tracker) {} + + ~SysObjReadCR() override { + try { + request_cleanup(); + } catch (const boost::container::length_error_t& e) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << + ": reference counted object mismatched, \"" << e.what() << + "\"" << dendl; + } + } + + void request_cleanup() override { + if (req) { + req->finish(); + req = NULL; + } + } + + int send_request(const DoutPrefixProvider *dpp) { + req = new RGWAsyncGetSystemObj(dpp, this, stack->create_completion_notifier(), svc, + objv_tracker, obj, false, false); + async_rados->queue(req); + return 0; + } + + int request_complete() { + int ret = req->get_ret_status(); + retcode = ret; + if (ret == -ENOENT && empty_on_enoent) { + *result = T(); + } else { + if (ret < 0) { + return ret; + } + if (objv_tracker) { // copy the updated version + *objv_tracker = req->objv_tracker; + } + try { + auto iter = req->bl.cbegin(); + if (iter.end()) { + // allow successful reads with empty buffers. ReadSyncStatus + // coroutines depend on this to be able to read without + // locking, because the cls lock from InitSyncStatus will + // create an empty object if it didn't exist + *result = T(); + } else { + decode(*result, iter); + } + } catch (buffer::error& err) { + return -EIO; + } + } + return handle_data(*result); + } + + virtual int handle_data(T& data) { + return 0; + } +}; + +template <class T> +class SysObjWriteCR : public RGWSimpleCoroutine { + const DoutPrefixProvider *dpp; + RGWAsyncRadosProcessor *async_rados; + RGWSI_SysObj *svc; + bufferlist bl; + rgw_raw_obj obj; + RGWObjVersionTracker *objv_tracker; + bool exclusive; + RGWAsyncPutSystemObj *req{nullptr}; + +public: + SysObjWriteCR(const DoutPrefixProvider *_dpp, + RGWAsyncRadosProcessor *_async_rados, RGWSI_SysObj *_svc, + const rgw_raw_obj& _obj, const T& _data, + RGWObjVersionTracker *objv_tracker = nullptr, + bool exclusive = false) + : RGWSimpleCoroutine(_svc->ctx()), dpp(_dpp), async_rados(_async_rados), + svc(_svc), obj(_obj), objv_tracker(objv_tracker), exclusive(exclusive) { + encode(_data, bl); + } + + ~SysObjWriteCR() override { + try { + request_cleanup(); + } catch (const boost::container::length_error_t& e) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << + ": reference counted object mismatched, \"" << e.what() << + "\"" << dendl; + } + } + + void request_cleanup() override { + if (req) { + req->finish(); + req = NULL; + } + } + + int send_request(const DoutPrefixProvider *dpp) override { + req = new RGWAsyncPutSystemObj(dpp, this, stack->create_completion_notifier(), + svc, objv_tracker, obj, exclusive, std::move(bl)); + async_rados->queue(req); + return 0; + } + + int request_complete() override { + if (objv_tracker) { // copy the updated version + *objv_tracker = req->objv_tracker; + } + return req->get_ret_status(); + } +}; +} + +/// read the mdlog history and use it to initialize the given cursor +class ReadHistoryCR : public RGWCoroutine { + const DoutPrefixProvider *dpp; + Svc svc; + Cursor *cursor; + RGWObjVersionTracker *objv_tracker; + RGWMetadataLogHistory state; + RGWAsyncRadosProcessor *async_processor; + + public: + ReadHistoryCR(const DoutPrefixProvider *dpp, + const Svc& svc, + Cursor *cursor, + RGWObjVersionTracker *objv_tracker) + : RGWCoroutine(svc.zone->ctx()), dpp(dpp), svc(svc), + cursor(cursor), + objv_tracker(objv_tracker), + async_processor(svc.rados->get_async_processor()) + {} + + int operate(const DoutPrefixProvider *dpp) { + reenter(this) { + yield { + rgw_raw_obj obj{svc.zone->get_zone_params().log_pool, + RGWMetadataLogHistory::oid}; + constexpr bool empty_on_enoent = false; + + using ReadCR = SysObjReadCR<RGWMetadataLogHistory>; + call(new ReadCR(dpp, async_processor, svc.sysobj, obj, + &state, empty_on_enoent, objv_tracker)); + } + if (retcode < 0) { + ldpp_dout(dpp, 1) << "failed to read mdlog history: " + << cpp_strerror(retcode) << dendl; + return set_cr_error(retcode); + } + *cursor = svc.mdlog->period_history->lookup(state.oldest_realm_epoch); + if (!*cursor) { + return set_cr_error(cursor->get_error()); + } + + ldpp_dout(dpp, 10) << "read mdlog history with oldest period id=" + << state.oldest_period_id << " realm_epoch=" + << state.oldest_realm_epoch << dendl; + return set_cr_done(); + } + return 0; + } +}; + +/// write the given cursor to the mdlog history +class WriteHistoryCR : public RGWCoroutine { + const DoutPrefixProvider *dpp; + Svc svc; + Cursor cursor; + RGWObjVersionTracker *objv; + RGWMetadataLogHistory state; + RGWAsyncRadosProcessor *async_processor; + + public: + WriteHistoryCR(const DoutPrefixProvider *dpp, + Svc& svc, + const Cursor& cursor, + RGWObjVersionTracker *objv) + : RGWCoroutine(svc.zone->ctx()), dpp(dpp), svc(svc), + cursor(cursor), objv(objv), + async_processor(svc.rados->get_async_processor()) + {} + + int operate(const DoutPrefixProvider *dpp) { + reenter(this) { + state.oldest_period_id = cursor.get_period().get_id(); + state.oldest_realm_epoch = cursor.get_epoch(); + + yield { + rgw_raw_obj obj{svc.zone->get_zone_params().log_pool, + RGWMetadataLogHistory::oid}; + + using WriteCR = SysObjWriteCR<RGWMetadataLogHistory>; + call(new WriteCR(dpp, async_processor, svc.sysobj, obj, state, objv)); + } + if (retcode < 0) { + ldpp_dout(dpp, 1) << "failed to write mdlog history: " + << cpp_strerror(retcode) << dendl; + return set_cr_error(retcode); + } + + ldpp_dout(dpp, 10) << "wrote mdlog history with oldest period id=" + << state.oldest_period_id << " realm_epoch=" + << state.oldest_realm_epoch << dendl; + return set_cr_done(); + } + return 0; + } +}; + +/// update the mdlog history to reflect trimmed logs +class TrimHistoryCR : public RGWCoroutine { + const DoutPrefixProvider *dpp; + Svc svc; + const Cursor cursor; //< cursor to trimmed period + RGWObjVersionTracker *objv; //< to prevent racing updates + Cursor next; //< target cursor for oldest log period + Cursor existing; //< existing cursor read from disk + + public: + TrimHistoryCR(const DoutPrefixProvider *dpp, const Svc& svc, Cursor cursor, RGWObjVersionTracker *objv) + : RGWCoroutine(svc.zone->ctx()), dpp(dpp), svc(svc), + cursor(cursor), objv(objv), next(cursor) { + next.next(); // advance past cursor + } + + int operate(const DoutPrefixProvider *dpp) { + reenter(this) { + // read an existing history, and write the new history if it's newer + yield call(new ReadHistoryCR(dpp, svc, &existing, objv)); + if (retcode < 0) { + return set_cr_error(retcode); + } + // reject older trims with ECANCELED + if (cursor.get_epoch() < existing.get_epoch()) { + ldpp_dout(dpp, 4) << "found oldest log epoch=" << existing.get_epoch() + << ", rejecting trim at epoch=" << cursor.get_epoch() << dendl; + return set_cr_error(-ECANCELED); + } + // overwrite with updated history + yield call(new WriteHistoryCR(dpp, svc, next, objv)); + if (retcode < 0) { + return set_cr_error(retcode); + } + return set_cr_done(); + } + return 0; + } +}; + +} // mdlog namespace + +// traverse all the way back to the beginning of the period history, and +// return a cursor to the first period in a fully attached history +Cursor RGWSI_MDLog::find_oldest_period(const DoutPrefixProvider *dpp, optional_yield y) +{ + auto cursor = period_history->get_current(); + + while (cursor) { + // advance to the period's predecessor + if (!cursor.has_prev()) { + auto& predecessor = cursor.get_period().get_predecessor(); + if (predecessor.empty()) { + // this is the first period, so our logs must start here + ldpp_dout(dpp, 10) << "find_oldest_period returning first " + "period " << cursor.get_period().get_id() << dendl; + return cursor; + } + // pull the predecessor and add it to our history + RGWPeriod period; + int r = period_puller->pull(dpp, predecessor, period, y); + if (r < 0) { + return cursor; + } + auto prev = period_history->insert(std::move(period)); + if (!prev) { + return prev; + } + ldpp_dout(dpp, 20) << "find_oldest_period advancing to " + "predecessor period " << predecessor << dendl; + ceph_assert(cursor.has_prev()); + } + cursor.prev(); + } + ldpp_dout(dpp, 10) << "find_oldest_period returning empty cursor" << dendl; + return cursor; +} + +Cursor RGWSI_MDLog::init_oldest_log_period(optional_yield y, const DoutPrefixProvider *dpp) +{ + // read the mdlog history + RGWMetadataLogHistory state; + RGWObjVersionTracker objv; + int ret = read_history(&state, &objv, y, dpp); + + if (ret == -ENOENT) { + // initialize the mdlog history and write it + ldpp_dout(dpp, 10) << "initializing mdlog history" << dendl; + auto cursor = find_oldest_period(dpp, y); + if (!cursor) { + return cursor; + } + // write the initial history + state.oldest_realm_epoch = cursor.get_epoch(); + state.oldest_period_id = cursor.get_period().get_id(); + + constexpr bool exclusive = true; // don't overwrite + int ret = write_history(dpp, state, &objv, y, exclusive); + if (ret < 0 && ret != -EEXIST) { + ldpp_dout(dpp, 1) << "failed to write mdlog history: " + << cpp_strerror(ret) << dendl; + return Cursor{ret}; + } + return cursor; + } else if (ret < 0) { + ldpp_dout(dpp, 1) << "failed to read mdlog history: " + << cpp_strerror(ret) << dendl; + return Cursor{ret}; + } + + // if it's already in the history, return it + auto cursor = period_history->lookup(state.oldest_realm_epoch); + if (cursor) { + return cursor; + } else { + cursor = find_oldest_period(dpp, y); + state.oldest_realm_epoch = cursor.get_epoch(); + state.oldest_period_id = cursor.get_period().get_id(); + ldpp_dout(dpp, 10) << "rewriting mdlog history" << dendl; + ret = write_history(dpp, state, &objv, y); + if (ret < 0 && ret != -ECANCELED) { + ldpp_dout(dpp, 1) << "failed to write mdlog history: " + << cpp_strerror(ret) << dendl; + return Cursor{ret}; + } + return cursor; + } + + // pull the oldest period by id + RGWPeriod period; + ret = period_puller->pull(dpp, state.oldest_period_id, period, y); + if (ret < 0) { + ldpp_dout(dpp, 1) << "failed to read period id=" << state.oldest_period_id + << " for mdlog history: " << cpp_strerror(ret) << dendl; + return Cursor{ret}; + } + // verify its realm_epoch + if (period.get_realm_epoch() != state.oldest_realm_epoch) { + ldpp_dout(dpp, 1) << "inconsistent mdlog history: read period id=" + << period.get_id() << " with realm_epoch=" << period.get_realm_epoch() + << ", expected realm_epoch=" << state.oldest_realm_epoch << dendl; + return Cursor{-EINVAL}; + } + // attach the period to our history + return period_history->attach(dpp, std::move(period), y); +} + +Cursor RGWSI_MDLog::read_oldest_log_period(optional_yield y, const DoutPrefixProvider *dpp) const +{ + RGWMetadataLogHistory state; + int ret = read_history(&state, nullptr, y, dpp); + if (ret < 0) { + ldpp_dout(dpp, 1) << "failed to read mdlog history: " + << cpp_strerror(ret) << dendl; + return Cursor{ret}; + } + + ldpp_dout(dpp, 10) << "read mdlog history with oldest period id=" + << state.oldest_period_id << " realm_epoch=" + << state.oldest_realm_epoch << dendl; + + return period_history->lookup(state.oldest_realm_epoch); +} + +RGWCoroutine* RGWSI_MDLog::read_oldest_log_period_cr(const DoutPrefixProvider *dpp, + Cursor *period, RGWObjVersionTracker *objv) const +{ + return new mdlog::ReadHistoryCR(dpp, svc, period, objv); +} + +RGWCoroutine* RGWSI_MDLog::trim_log_period_cr(const DoutPrefixProvider *dpp, + Cursor period, RGWObjVersionTracker *objv) const +{ + return new mdlog::TrimHistoryCR(dpp, svc, period, objv); +} + +RGWMetadataLog* RGWSI_MDLog::get_log(const std::string& period) +{ + // construct the period's log in place if it doesn't exist + auto insert = md_logs.emplace(std::piecewise_construct, + std::forward_as_tuple(period), + std::forward_as_tuple(cct, svc.zone, svc.cls, period)); + return &insert.first->second; +} + +int RGWSI_MDLog::add_entry(const DoutPrefixProvider *dpp, const string& hash_key, const string& section, const string& key, bufferlist& bl) +{ + ceph_assert(current_log); // must have called init() + return current_log->add_entry(dpp, hash_key, section, key, bl); +} + +int RGWSI_MDLog::get_shard_id(const string& hash_key, int *shard_id) +{ + ceph_assert(current_log); // must have called init() + return current_log->get_shard_id(hash_key, shard_id); +} + +int RGWSI_MDLog::pull_period(const DoutPrefixProvider *dpp, const std::string& period_id, RGWPeriod& period, + optional_yield y) +{ + return period_puller->pull(dpp, period_id, period, y); +} + diff --git a/src/rgw/services/svc_mdlog.h b/src/rgw/services/svc_mdlog.h new file mode 100644 index 000000000..703d6f605 --- /dev/null +++ b/src/rgw/services/svc_mdlog.h @@ -0,0 +1,118 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" +#include "rgw_period_history.h" +#include "rgw_period_puller.h" + +#include "svc_meta_be.h" + + +class RGWMetadataLog; +class RGWMetadataLogHistory; +class RGWCoroutine; + +class RGWSI_Zone; +class RGWSI_SysObj; +class RGWSI_RADOS; + +namespace mdlog { + class ReadHistoryCR; + class WriteHistoryCR; +} + +class RGWSI_MDLog : public RGWServiceInstance +{ + friend class mdlog::ReadHistoryCR; + friend class mdlog::WriteHistoryCR; + + // maintain a separate metadata log for each period + std::map<std::string, RGWMetadataLog> md_logs; + + // use the current period's log for mutating operations + RGWMetadataLog* current_log{nullptr}; + + bool run_sync; + + // pulls missing periods for period_history + std::unique_ptr<RGWPeriodPuller> period_puller; + // maintains a connected history of periods + std::unique_ptr<RGWPeriodHistory> period_history; + +public: + RGWSI_MDLog(CephContext *cct, bool run_sync); + virtual ~RGWSI_MDLog(); + + struct Svc { + RGWSI_RADOS *rados{nullptr}; + RGWSI_Zone *zone{nullptr}; + RGWSI_SysObj *sysobj{nullptr}; + RGWSI_MDLog *mdlog{nullptr}; + RGWSI_Cls *cls{nullptr}; + } svc; + + int init(RGWSI_RADOS *_rados_svc, + RGWSI_Zone *_zone_svc, + RGWSI_SysObj *_sysobj_svc, + RGWSI_Cls *_cls_svc); + + int do_start(optional_yield y, const DoutPrefixProvider *dpp) override; + + // traverse all the way back to the beginning of the period history, and + // return a cursor to the first period in a fully attached history + RGWPeriodHistory::Cursor find_oldest_period(const DoutPrefixProvider *dpp, optional_yield y); + + /// initialize the oldest log period if it doesn't exist, and attach it to + /// our current history + RGWPeriodHistory::Cursor init_oldest_log_period(optional_yield y, const DoutPrefixProvider *dpp); + + /// read the oldest log period, and return a cursor to it in our existing + /// period history + RGWPeriodHistory::Cursor read_oldest_log_period(optional_yield y, const DoutPrefixProvider *dpp) const; + + /// read the oldest log period asynchronously and write its result to the + /// given cursor pointer + RGWCoroutine* read_oldest_log_period_cr(const DoutPrefixProvider *dpp, + RGWPeriodHistory::Cursor *period, + RGWObjVersionTracker *objv) const; + + /// try to advance the oldest log period when the given period is trimmed, + /// using a rados lock to provide atomicity + RGWCoroutine* trim_log_period_cr(const DoutPrefixProvider *dpp, + RGWPeriodHistory::Cursor period, + RGWObjVersionTracker *objv) const; + int read_history(RGWMetadataLogHistory *state, RGWObjVersionTracker *objv_tracker,optional_yield y, const DoutPrefixProvider *dpp) const; + int write_history(const DoutPrefixProvider *dpp, + const RGWMetadataLogHistory& state, + RGWObjVersionTracker *objv_tracker, + optional_yield y, bool exclusive = false); + + int add_entry(const DoutPrefixProvider *dpp, const std::string& hash_key, const std::string& section, const std::string& key, bufferlist& bl); + + int get_shard_id(const std::string& hash_key, int *shard_id); + + RGWPeriodHistory *get_period_history() { + return period_history.get(); + } + + int pull_period(const DoutPrefixProvider *dpp, const std::string& period_id, RGWPeriod& period, optional_yield y); + + /// find or create the metadata log for the given period + RGWMetadataLog* get_log(const std::string& period); +}; + diff --git a/src/rgw/services/svc_meta.cc b/src/rgw/services/svc_meta.cc new file mode 100644 index 000000000..735c39f85 --- /dev/null +++ b/src/rgw/services/svc_meta.cc @@ -0,0 +1,46 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + + +#include "svc_meta.h" + +#include "rgw_metadata.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +RGWSI_Meta::RGWSI_Meta(CephContext *cct) : RGWServiceInstance(cct) { +} + +RGWSI_Meta::~RGWSI_Meta() {} + +void RGWSI_Meta::init(RGWSI_SysObj *_sysobj_svc, + RGWSI_MDLog *_mdlog_svc, + vector<RGWSI_MetaBackend *>& _be_svc) +{ + sysobj_svc = _sysobj_svc; + mdlog_svc = _mdlog_svc; + + for (auto& be : _be_svc) { + be_svc[be->get_type()] = be; + } +} + +int RGWSI_Meta::create_be_handler(RGWSI_MetaBackend::Type be_type, + RGWSI_MetaBackend_Handler **phandler) +{ + auto iter = be_svc.find(be_type); + if (iter == be_svc.end()) { + ldout(cct, 0) << __func__ << "(): ERROR: backend type not found" << dendl; + return -EINVAL; + } + + auto handler = iter->second->alloc_be_handler(); + + be_handlers.emplace_back(handler); + *phandler = handler; + + return 0; +} + diff --git a/src/rgw/services/svc_meta.h b/src/rgw/services/svc_meta.h new file mode 100644 index 000000000..b398e27fd --- /dev/null +++ b/src/rgw/services/svc_meta.h @@ -0,0 +1,48 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "svc_meta_be.h" + +#include "rgw_service.h" + + +class RGWMetadataLog; +class RGWCoroutine; + + +class RGWSI_Meta : public RGWServiceInstance +{ + RGWSI_SysObj *sysobj_svc{nullptr}; + RGWSI_MDLog *mdlog_svc{nullptr}; + + std::map<RGWSI_MetaBackend::Type, RGWSI_MetaBackend *> be_svc; + + std::vector<std::unique_ptr<RGWSI_MetaBackend_Handler> > be_handlers; + +public: + RGWSI_Meta(CephContext *cct); + ~RGWSI_Meta(); + + void init(RGWSI_SysObj *_sysobj_svc, + RGWSI_MDLog *_mdlog_svc, + std::vector<RGWSI_MetaBackend *>& _be_svc); + + int create_be_handler(RGWSI_MetaBackend::Type be_type, + RGWSI_MetaBackend_Handler **phandler); +}; + diff --git a/src/rgw/services/svc_meta_be.cc b/src/rgw/services/svc_meta_be.cc new file mode 100644 index 000000000..2cb0365c8 --- /dev/null +++ b/src/rgw/services/svc_meta_be.cc @@ -0,0 +1,193 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + + +#include "svc_meta_be.h" + +#include "rgw_mdlog.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +RGWSI_MetaBackend::Context::~Context() {} // needed, even though destructor is pure virtual +RGWSI_MetaBackend::Module::~Module() {} // ditto +RGWSI_MetaBackend::PutParams::~PutParams() {} // ... +RGWSI_MetaBackend::GetParams::~GetParams() {} // ... +RGWSI_MetaBackend::RemoveParams::~RemoveParams() {} // ... + +int RGWSI_MetaBackend::pre_modify(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, + RGWMDLogStatus op_type, + optional_yield y) +{ + /* if write version has not been set, and there's a read version, set it so that we can + * log it + */ + if (objv_tracker && + objv_tracker->read_version.ver && !objv_tracker->write_version.ver) { + objv_tracker->write_version = objv_tracker->read_version; + objv_tracker->write_version.ver++; + } + + return 0; +} + +int RGWSI_MetaBackend::post_modify(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, int ret, + optional_yield y) +{ + return ret; +} + +int RGWSI_MetaBackend::prepare_mutate(RGWSI_MetaBackend::Context *ctx, + const string& key, + const real_time& mtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + real_time orig_mtime; + + int ret = call_with_get_params(&orig_mtime, [&](GetParams& params) { + return get_entry(ctx, key, params, objv_tracker, y, dpp); + }); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + + if (objv_tracker->write_version.tag.empty()) { + if (objv_tracker->read_version.tag.empty()) { + objv_tracker->generate_new_write_ver(cct); + } else { + objv_tracker->write_version = objv_tracker->read_version; + objv_tracker->write_version.ver++; + } + } + return 0; +} + +int RGWSI_MetaBackend::do_mutate(RGWSI_MetaBackend::Context *ctx, + const string& key, + const ceph::real_time& mtime, + RGWObjVersionTracker *objv_tracker, + RGWMDLogStatus op_type, + optional_yield y, + std::function<int()> f, + bool generic_prepare, + const DoutPrefixProvider *dpp) +{ + int ret; + + if (generic_prepare) { + ret = prepare_mutate(ctx, key, mtime, objv_tracker, y, dpp); + if (ret < 0 || + ret == STATUS_NO_APPLY) { + return ret; + } + } + + RGWMetadataLogData log_data; + ret = pre_modify(dpp, ctx, key, log_data, objv_tracker, op_type, y); + if (ret < 0) { + return ret; + } + + ret = f(); + + /* cascading ret into post_modify() */ + + ret = post_modify(dpp, ctx, key, log_data, objv_tracker, ret, y); + if (ret < 0) + return ret; + + return 0; +} + +int RGWSI_MetaBackend::get(Context *ctx, + const string& key, + GetParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + bool get_raw_attrs) +{ + return get_entry(ctx, key, params, objv_tracker, y, dpp, get_raw_attrs); +} + +int RGWSI_MetaBackend::put(Context *ctx, + const string& key, + PutParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + std::function<int()> f = [&]() { + return put_entry(dpp, ctx, key, params, objv_tracker, y); + }; + + return do_mutate(ctx, key, params.mtime, objv_tracker, + MDLOG_STATUS_WRITE, + y, + f, + false, + dpp); +} + +int RGWSI_MetaBackend::remove(Context *ctx, + const string& key, + RemoveParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + std::function<int()> f = [&]() { + return remove_entry(dpp, ctx, key, params, objv_tracker, y); + }; + + return do_mutate(ctx, key, params.mtime, objv_tracker, + MDLOG_STATUS_REMOVE, + y, + f, + false, + dpp); +} + +int RGWSI_MetaBackend::mutate(Context *ctx, + const std::string& key, + MutateParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + std::function<int()> f, + const DoutPrefixProvider *dpp) +{ + return do_mutate(ctx, key, params.mtime, objv_tracker, + params.op_type, y, + f, + false, + dpp); +} + +int RGWSI_MetaBackend_Handler::call(std::optional<RGWSI_MetaBackend_CtxParams> bectx_params, + std::function<int(Op *)> f) +{ + return be->call(bectx_params, [&](RGWSI_MetaBackend::Context *ctx) { + ctx->init(this); + Op op(be, ctx); + return f(&op); + }); +} + +RGWSI_MetaBackend_Handler::Op_ManagedCtx::Op_ManagedCtx(RGWSI_MetaBackend_Handler *handler) : Op(handler->be, handler->be->alloc_ctx()) +{ + auto c = ctx(); + c->init(handler); + pctx.reset(c); +} + diff --git a/src/rgw/services/svc_meta_be.h b/src/rgw/services/svc_meta_be.h new file mode 100644 index 000000000..97267a4e7 --- /dev/null +++ b/src/rgw/services/svc_meta_be.h @@ -0,0 +1,294 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "svc_meta_be_params.h" + +#include "rgw_service.h" +#include "rgw_mdlog_types.h" + +class RGWMetadataLogData; + +class RGWSI_MDLog; +class RGWSI_Meta; +class RGWObjVersionTracker; +class RGWSI_MetaBackend_Handler; + +class RGWSI_MetaBackend : public RGWServiceInstance +{ + friend class RGWSI_Meta; +public: + class Module; + class Context; +protected: + RGWSI_MDLog *mdlog_svc{nullptr}; + + void base_init(RGWSI_MDLog *_mdlog_svc) { + mdlog_svc = _mdlog_svc; + } + + int prepare_mutate(RGWSI_MetaBackend::Context *ctx, + const std::string& key, + const ceph::real_time& mtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp); + + virtual int do_mutate(Context *ctx, + const std::string& key, + const ceph::real_time& mtime, RGWObjVersionTracker *objv_tracker, + RGWMDLogStatus op_type, + optional_yield y, + std::function<int()> f, + bool generic_prepare, + const DoutPrefixProvider *dpp); + + virtual int pre_modify(const DoutPrefixProvider *dpp, + Context *ctx, + const std::string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, + RGWMDLogStatus op_type, + optional_yield y); + virtual int post_modify(const DoutPrefixProvider *dpp, + Context *ctx, + const std::string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, int ret, + optional_yield y); +public: + class Module { + /* + * Backend specialization module + */ + public: + virtual ~Module() = 0; + }; + + using ModuleRef = std::shared_ptr<Module>; + + struct Context { /* + * A single metadata operation context. Will be holding info about + * backend and operation itself; operation might span multiple backend + * calls. + */ + virtual ~Context() = 0; + + virtual void init(RGWSI_MetaBackend_Handler *h) = 0; + }; + + virtual Context *alloc_ctx() = 0; + + struct PutParams { + ceph::real_time mtime; + + PutParams() {} + PutParams(const ceph::real_time& _mtime) : mtime(_mtime) {} + virtual ~PutParams() = 0; + }; + + struct GetParams { + GetParams() {} + GetParams(ceph::real_time *_pmtime) : pmtime(_pmtime) {} + virtual ~GetParams(); + + ceph::real_time *pmtime{nullptr}; + }; + + struct RemoveParams { + virtual ~RemoveParams() = 0; + + ceph::real_time mtime; + }; + + struct MutateParams { + ceph::real_time mtime; + RGWMDLogStatus op_type; + + MutateParams() {} + MutateParams(const ceph::real_time& _mtime, + RGWMDLogStatus _op_type) : mtime(_mtime), op_type(_op_type) {} + virtual ~MutateParams() {} + }; + + enum Type { + MDBE_SOBJ = 0, + MDBE_OTP = 1, + }; + + RGWSI_MetaBackend(CephContext *cct) : RGWServiceInstance(cct) {} + virtual ~RGWSI_MetaBackend() {} + + virtual Type get_type() = 0; + + virtual RGWSI_MetaBackend_Handler *alloc_be_handler() = 0; + virtual int call_with_get_params(ceph::real_time *pmtime, std::function<int(RGWSI_MetaBackend::GetParams&)>) = 0; + + /* these should be implemented by backends */ + virtual int get_entry(RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWSI_MetaBackend::GetParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + bool get_raw_attrs=false) = 0; + virtual int put_entry(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWSI_MetaBackend::PutParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y) = 0; + virtual int remove_entry(const DoutPrefixProvider *dpp, + Context *ctx, + const std::string& key, + RGWSI_MetaBackend::RemoveParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y) = 0; + + virtual int list_init(const DoutPrefixProvider *dpp, RGWSI_MetaBackend::Context *ctx, const std::string& marker) = 0; + virtual int list_next(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + int max, std::list<std::string> *keys, + bool *truncated) = 0; + virtual int list_get_marker(RGWSI_MetaBackend::Context *ctx, + std::string *marker) = 0; + + int call(std::function<int(RGWSI_MetaBackend::Context *)> f) { + return call(std::nullopt, f); + } + + virtual int call(std::optional<RGWSI_MetaBackend_CtxParams> opt, + std::function<int(RGWSI_MetaBackend::Context *)> f) = 0; + + virtual int get_shard_id(RGWSI_MetaBackend::Context *ctx, + const std::string& key, + int *shard_id) = 0; + + /* higher level */ + virtual int get(Context *ctx, + const std::string& key, + GetParams ¶ms, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + bool get_raw_attrs=false); + + virtual int put(Context *ctx, + const std::string& key, + PutParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp); + + virtual int remove(Context *ctx, + const std::string& key, + RemoveParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp); + + virtual int mutate(Context *ctx, + const std::string& key, + MutateParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + std::function<int()> f, + const DoutPrefixProvider *dpp); +}; + +class RGWSI_MetaBackend_Handler { + RGWSI_MetaBackend *be{nullptr}; + +public: + class Op { + friend class RGWSI_MetaBackend_Handler; + + RGWSI_MetaBackend *be; + RGWSI_MetaBackend::Context *be_ctx; + + Op(RGWSI_MetaBackend *_be, + RGWSI_MetaBackend::Context *_ctx) : be(_be), be_ctx(_ctx) {} + + public: + RGWSI_MetaBackend::Context *ctx() { + return be_ctx; + } + + int get(const std::string& key, + RGWSI_MetaBackend::GetParams ¶ms, + RGWObjVersionTracker *objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) { + return be->get(be_ctx, key, params, objv_tracker, y, dpp); + } + + int put(const std::string& key, + RGWSI_MetaBackend::PutParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) { + return be->put(be_ctx, key, params, objv_tracker, y, dpp); + } + + int remove(const std::string& key, + RGWSI_MetaBackend::RemoveParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) { + return be->remove(be_ctx, key, params, objv_tracker, y, dpp); + } + + int mutate(const std::string& key, + RGWSI_MetaBackend::MutateParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + std::function<int()> f, + const DoutPrefixProvider *dpp) { + return be->mutate(be_ctx, key, params, objv_tracker, y, f, dpp); + } + + int list_init(const DoutPrefixProvider *dpp, const std::string& marker) { + return be->list_init(dpp, be_ctx, marker); + } + int list_next(const DoutPrefixProvider *dpp, int max, std::list<std::string> *keys, + bool *truncated) { + return be->list_next(dpp, be_ctx, max, keys, truncated); + } + int list_get_marker(std::string *marker) { + return be->list_get_marker(be_ctx, marker); + } + + int get_shard_id(const std::string& key, int *shard_id) { + return be->get_shard_id(be_ctx, key, shard_id); + } + }; + + class Op_ManagedCtx : public Op { + std::unique_ptr<RGWSI_MetaBackend::Context> pctx; + public: + Op_ManagedCtx(RGWSI_MetaBackend_Handler *handler); + }; + + RGWSI_MetaBackend_Handler(RGWSI_MetaBackend *_be) : be(_be) {} + virtual ~RGWSI_MetaBackend_Handler() {} + + int call(std::function<int(Op *)> f) { + return call(std::nullopt, f); + } + + virtual int call(std::optional<RGWSI_MetaBackend_CtxParams> bectx_params, + std::function<int(Op *)> f); +}; + diff --git a/src/rgw/services/svc_meta_be_otp.cc b/src/rgw/services/svc_meta_be_otp.cc new file mode 100644 index 000000000..3cabeb9d0 --- /dev/null +++ b/src/rgw/services/svc_meta_be_otp.cc @@ -0,0 +1,73 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_meta_be_otp.h" + +#include "rgw_tools.h" +#include "rgw_metadata.h" +#include "rgw_mdlog.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +RGWSI_MetaBackend_OTP::RGWSI_MetaBackend_OTP(CephContext *cct) : RGWSI_MetaBackend_SObj(cct) { +} + +RGWSI_MetaBackend_OTP::~RGWSI_MetaBackend_OTP() { +} + +string RGWSI_MetaBackend_OTP::get_meta_key(const rgw_user& user) +{ + return string("otp:user:") + user.to_str(); +} + +RGWSI_MetaBackend_Handler *RGWSI_MetaBackend_OTP::alloc_be_handler() +{ + return new RGWSI_MetaBackend_Handler_OTP(this); +} + +RGWSI_MetaBackend::Context *RGWSI_MetaBackend_OTP::alloc_ctx() +{ + return new Context_OTP; +} + +int RGWSI_MetaBackend_OTP::call_with_get_params(ceph::real_time *pmtime, std::function<int(RGWSI_MetaBackend::GetParams&)> cb) +{ + otp_devices_list_t devices; + RGWSI_MBOTP_GetParams params; + params.pdevices = &devices; + params.pmtime = pmtime; + return cb(params); +} + +int RGWSI_MetaBackend_OTP::get_entry(RGWSI_MetaBackend::Context *_ctx, + const string& key, + RGWSI_MetaBackend::GetParams& _params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + bool get_raw_attrs) +{ + RGWSI_MBOTP_GetParams& params = static_cast<RGWSI_MBOTP_GetParams&>(_params); + + int r = cls_svc->mfa.list_mfa(dpp, key, params.pdevices, objv_tracker, params.pmtime, y); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_MetaBackend_OTP::put_entry(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *_ctx, + const string& key, + RGWSI_MetaBackend::PutParams& _params, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + RGWSI_MBOTP_PutParams& params = static_cast<RGWSI_MBOTP_PutParams&>(_params); + + return cls_svc->mfa.set_mfa(dpp, key, params.devices, true, objv_tracker, params.mtime, y); +} + diff --git a/src/rgw/services/svc_meta_be_otp.h b/src/rgw/services/svc_meta_be_otp.h new file mode 100644 index 000000000..7bd9cf652 --- /dev/null +++ b/src/rgw/services/svc_meta_be_otp.h @@ -0,0 +1,89 @@ + + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_cls.h" +#include "svc_meta_be.h" +#include "svc_meta_be_sobj.h" +#include "svc_sys_obj.h" + + +using RGWSI_MBOTP_Handler_Module = RGWSI_MBSObj_Handler_Module; +using RGWSI_MetaBackend_Handler_OTP = RGWSI_MetaBackend_Handler_SObj; + +using otp_devices_list_t = std::list<rados::cls::otp::otp_info_t>; + +struct RGWSI_MBOTP_GetParams : public RGWSI_MetaBackend::GetParams { + otp_devices_list_t *pdevices{nullptr}; +}; + +struct RGWSI_MBOTP_PutParams : public RGWSI_MetaBackend::PutParams { + otp_devices_list_t devices; +}; + +using RGWSI_MBOTP_RemoveParams = RGWSI_MBSObj_RemoveParams; + +class RGWSI_MetaBackend_OTP : public RGWSI_MetaBackend_SObj +{ + RGWSI_Cls *cls_svc{nullptr}; + +public: + struct Context_OTP : public RGWSI_MetaBackend_SObj::Context_SObj { + otp_devices_list_t devices; + }; + + RGWSI_MetaBackend_OTP(CephContext *cct); + virtual ~RGWSI_MetaBackend_OTP(); + + RGWSI_MetaBackend::Type get_type() { + return MDBE_OTP; + } + + static std::string get_meta_key(const rgw_user& user); + + void init(RGWSI_SysObj *_sysobj_svc, + RGWSI_MDLog *_mdlog_svc, + RGWSI_Cls *_cls_svc) { + RGWSI_MetaBackend_SObj::init(_sysobj_svc, _mdlog_svc); + cls_svc = _cls_svc; + } + + RGWSI_MetaBackend_Handler *alloc_be_handler() override; + RGWSI_MetaBackend::Context *alloc_ctx() override; + + int call_with_get_params(ceph::real_time *pmtime, std::function<int(RGWSI_MetaBackend::GetParams&)> cb) override; + + int get_entry(RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWSI_MetaBackend::GetParams& _params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + bool get_raw_attrs=false); + int put_entry(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWSI_MetaBackend::PutParams& _params, + RGWObjVersionTracker *objv_tracker, + optional_yield y); +}; + + diff --git a/src/rgw/services/svc_meta_be_params.h b/src/rgw/services/svc_meta_be_params.h new file mode 100644 index 000000000..445f6e188 --- /dev/null +++ b/src/rgw/services/svc_meta_be_params.h @@ -0,0 +1,25 @@ + + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include <variant> + +struct RGWSI_MetaBackend_CtxParams_SObj {}; + +using RGWSI_MetaBackend_CtxParams = std::variant<RGWSI_MetaBackend_CtxParams_SObj>; diff --git a/src/rgw/services/svc_meta_be_sobj.cc b/src/rgw/services/svc_meta_be_sobj.cc new file mode 100644 index 000000000..c0ff402fc --- /dev/null +++ b/src/rgw/services/svc_meta_be_sobj.cc @@ -0,0 +1,246 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_meta_be_sobj.h" +#include "svc_meta_be_params.h" +#include "svc_mdlog.h" + +#include "rgw_tools.h" +#include "rgw_metadata.h" +#include "rgw_mdlog.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +RGWSI_MetaBackend_SObj::RGWSI_MetaBackend_SObj(CephContext *cct) : RGWSI_MetaBackend(cct) { +} + +RGWSI_MetaBackend_SObj::~RGWSI_MetaBackend_SObj() { +} + +RGWSI_MetaBackend_Handler *RGWSI_MetaBackend_SObj::alloc_be_handler() +{ + return new RGWSI_MetaBackend_Handler_SObj(this); +} + +RGWSI_MetaBackend::Context *RGWSI_MetaBackend_SObj::alloc_ctx() +{ + return new Context_SObj; +} + +int RGWSI_MetaBackend_SObj::pre_modify(const DoutPrefixProvider *dpp, RGWSI_MetaBackend::Context *_ctx, + const string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, + RGWMDLogStatus op_type, + optional_yield y) +{ + auto ctx = static_cast<Context_SObj *>(_ctx); + int ret = RGWSI_MetaBackend::pre_modify(dpp, ctx, key, log_data, + objv_tracker, op_type, + y); + if (ret < 0) { + return ret; + } + + /* if write version has not been set, and there's a read version, set it so that we can + * log it + */ + if (objv_tracker) { + log_data.read_version = objv_tracker->read_version; + log_data.write_version = objv_tracker->write_version; + } + + log_data.status = op_type; + + bufferlist logbl; + encode(log_data, logbl); + + ret = mdlog_svc->add_entry(dpp, ctx->module->get_hash_key(key), ctx->module->get_section(), key, logbl); + if (ret < 0) + return ret; + + return 0; +} + +int RGWSI_MetaBackend_SObj::post_modify(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *_ctx, + const string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, int ret, + optional_yield y) +{ + auto ctx = static_cast<Context_SObj *>(_ctx); + if (ret >= 0) + log_data.status = MDLOG_STATUS_COMPLETE; + else + log_data.status = MDLOG_STATUS_ABORT; + + bufferlist logbl; + encode(log_data, logbl); + + int r = mdlog_svc->add_entry(dpp, ctx->module->get_hash_key(key), ctx->module->get_section(), key, logbl); + if (ret < 0) + return ret; + + if (r < 0) + return r; + + return RGWSI_MetaBackend::post_modify(dpp, ctx, key, log_data, objv_tracker, ret, y); +} + +int RGWSI_MetaBackend_SObj::get_shard_id(RGWSI_MetaBackend::Context *_ctx, + const std::string& key, + int *shard_id) +{ + auto ctx = static_cast<Context_SObj *>(_ctx); + *shard_id = mdlog_svc->get_shard_id(ctx->module->get_hash_key(key), shard_id); + return 0; +} + +int RGWSI_MetaBackend_SObj::call(std::optional<RGWSI_MetaBackend_CtxParams> opt, + std::function<int(RGWSI_MetaBackend::Context *)> f) +{ + RGWSI_MetaBackend_SObj::Context_SObj ctx; + return f(&ctx); +} + +void RGWSI_MetaBackend_SObj::Context_SObj::init(RGWSI_MetaBackend_Handler *h) +{ + RGWSI_MetaBackend_Handler_SObj *handler = static_cast<RGWSI_MetaBackend_Handler_SObj *>(h); + module = handler->module; +} + +int RGWSI_MetaBackend_SObj::call_with_get_params(ceph::real_time *pmtime, std::function<int(RGWSI_MetaBackend::GetParams&)> cb) +{ + bufferlist bl; + RGWSI_MBSObj_GetParams params; + params.pmtime = pmtime; + params.pbl = &bl; + return cb(params); +} + +int RGWSI_MetaBackend_SObj::get_entry(RGWSI_MetaBackend::Context *_ctx, + const string& key, + GetParams& _params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + bool get_raw_attrs) +{ + RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx); + RGWSI_MBSObj_GetParams& params = static_cast<RGWSI_MBSObj_GetParams&>(_params); + + rgw_pool pool; + string oid; + ctx->module->get_pool_and_oid(key, &pool, &oid); + + int ret = 0; + ret = rgw_get_system_obj(sysobj_svc, pool, oid, *params.pbl, + objv_tracker, params.pmtime, + y, dpp, + params.pattrs, params.cache_info, + params.refresh_version, get_raw_attrs); + + return ret; +} + +int RGWSI_MetaBackend_SObj::put_entry(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *_ctx, + const string& key, + PutParams& _params, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx); + RGWSI_MBSObj_PutParams& params = static_cast<RGWSI_MBSObj_PutParams&>(_params); + + rgw_pool pool; + string oid; + ctx->module->get_pool_and_oid(key, &pool, &oid); + + return rgw_put_system_obj(dpp, sysobj_svc, pool, oid, params.bl, params.exclusive, + objv_tracker, params.mtime, y, params.pattrs); +} + +int RGWSI_MetaBackend_SObj::remove_entry(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *_ctx, + const string& key, + RemoveParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx); + + rgw_pool pool; + string oid; + ctx->module->get_pool_and_oid(key, &pool, &oid); + rgw_raw_obj k(pool, oid); + + auto sysobj = sysobj_svc->get_obj(k); + return sysobj.wop() + .set_objv_tracker(objv_tracker) + .remove(dpp, y); +} + +int RGWSI_MetaBackend_SObj::list_init(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *_ctx, + const string& marker) +{ + RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx); + + rgw_pool pool; + + string no_key; + ctx->module->get_pool_and_oid(no_key, &pool, nullptr); + + ctx->list.pool = sysobj_svc->get_pool(pool); + ctx->list.op.emplace(ctx->list.pool->op()); + + string prefix = ctx->module->get_oid_prefix(); + ctx->list.op->init(dpp, marker, prefix); + + return 0; +} + +int RGWSI_MetaBackend_SObj::list_next(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *_ctx, + int max, list<string> *keys, + bool *truncated) +{ + RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx); + + vector<string> oids; + + keys->clear(); + + int ret = ctx->list.op->get_next(dpp, max, &oids, truncated); + if (ret < 0 && ret != -ENOENT) + return ret; + if (ret == -ENOENT) { + if (truncated) + *truncated = false; + return 0; + } + + auto module = ctx->module; + + for (auto& o : oids) { + if (!module->is_valid_oid(o)) { + continue; + } + keys->emplace_back(module->oid_to_key(o)); + } + + return 0; +} + +int RGWSI_MetaBackend_SObj::list_get_marker(RGWSI_MetaBackend::Context *_ctx, + string *marker) +{ + RGWSI_MetaBackend_SObj::Context_SObj *ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx); + + return ctx->list.op->get_marker(marker); +} + diff --git a/src/rgw/services/svc_meta_be_sobj.h b/src/rgw/services/svc_meta_be_sobj.h new file mode 100644 index 000000000..304afc8bf --- /dev/null +++ b/src/rgw/services/svc_meta_be_sobj.h @@ -0,0 +1,194 @@ + + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_meta_be.h" +#include "svc_sys_obj.h" + + +class RGWSI_MBSObj_Handler_Module : public RGWSI_MetaBackend::Module { +protected: + std::string section; +public: + RGWSI_MBSObj_Handler_Module(const std::string& _section) : section(_section) {} + virtual void get_pool_and_oid(const std::string& key, rgw_pool *pool, std::string *oid) = 0; + virtual const std::string& get_oid_prefix() = 0; + virtual std::string key_to_oid(const std::string& key) = 0; + virtual bool is_valid_oid(const std::string& oid) = 0; + virtual std::string oid_to_key(const std::string& oid) = 0; + + const std::string& get_section() { + return section; + } + + /* key to use for hashing entries for log shard placement */ + virtual std::string get_hash_key(const std::string& key) { + return section + ":" + key; + } +}; + +struct RGWSI_MBSObj_GetParams : public RGWSI_MetaBackend::GetParams { + bufferlist *pbl{nullptr}; + std::map<std::string, bufferlist> *pattrs{nullptr}; + rgw_cache_entry_info *cache_info{nullptr}; + boost::optional<obj_version> refresh_version; + + RGWSI_MBSObj_GetParams() {} + RGWSI_MBSObj_GetParams(bufferlist *_pbl, + std::map<std::string, bufferlist> *_pattrs, + ceph::real_time *_pmtime) : RGWSI_MetaBackend::GetParams(_pmtime), + pbl(_pbl), + pattrs(_pattrs) {} + + RGWSI_MBSObj_GetParams& set_cache_info(rgw_cache_entry_info *_cache_info) { + cache_info = _cache_info; + return *this; + } + RGWSI_MBSObj_GetParams& set_refresh_version(boost::optional<obj_version>& _refresh_version) { + refresh_version = _refresh_version; + return *this; + } +}; + +struct RGWSI_MBSObj_PutParams : public RGWSI_MetaBackend::PutParams { + bufferlist bl; + std::map<std::string, bufferlist> *pattrs{nullptr}; + bool exclusive{false}; + + RGWSI_MBSObj_PutParams() {} + RGWSI_MBSObj_PutParams(std::map<std::string, bufferlist> *_pattrs, + const ceph::real_time& _mtime) : RGWSI_MetaBackend::PutParams(_mtime), + pattrs(_pattrs) {} + RGWSI_MBSObj_PutParams(bufferlist& _bl, + std::map<std::string, bufferlist> *_pattrs, + const ceph::real_time& _mtime, + bool _exclusive) : RGWSI_MetaBackend::PutParams(_mtime), + bl(_bl), + pattrs(_pattrs), + exclusive(_exclusive) {} +}; + +struct RGWSI_MBSObj_RemoveParams : public RGWSI_MetaBackend::RemoveParams { +}; + +class RGWSI_MetaBackend_SObj : public RGWSI_MetaBackend +{ +protected: + RGWSI_SysObj *sysobj_svc{nullptr}; + +public: + struct Context_SObj : public RGWSI_MetaBackend::Context { + RGWSI_MBSObj_Handler_Module *module{nullptr}; + struct _list { + std::optional<RGWSI_SysObj::Pool> pool; + std::optional<RGWSI_SysObj::Pool::Op> op; + } list; + + void init(RGWSI_MetaBackend_Handler *h) override; + }; + + RGWSI_MetaBackend_SObj(CephContext *cct); + virtual ~RGWSI_MetaBackend_SObj(); + + RGWSI_MetaBackend::Type get_type() { + return MDBE_SOBJ; + } + + void init(RGWSI_SysObj *_sysobj_svc, + RGWSI_MDLog *_mdlog_svc) { + base_init(_mdlog_svc); + sysobj_svc = _sysobj_svc; + } + + RGWSI_MetaBackend_Handler *alloc_be_handler() override; + RGWSI_MetaBackend::Context *alloc_ctx() override; + + + int call_with_get_params(ceph::real_time *pmtime, std::function<int(RGWSI_MetaBackend::GetParams&)> cb) override; + + int pre_modify(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, + RGWMDLogStatus op_type, + optional_yield y); + int post_modify(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWMetadataLogData& log_data, + RGWObjVersionTracker *objv_tracker, int ret, + optional_yield y); + + int get_entry(RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWSI_MetaBackend::GetParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + bool get_raw_attrs=false) override; + int put_entry(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWSI_MetaBackend::PutParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y) override; + int remove_entry(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const std::string& key, + RGWSI_MetaBackend::RemoveParams& params, + RGWObjVersionTracker *objv_tracker, + optional_yield y) override; + + int list_init(const DoutPrefixProvider *dpp, RGWSI_MetaBackend::Context *_ctx, const std::string& marker) override; + int list_next(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *_ctx, + int max, std::list<std::string> *keys, + bool *truncated) override; + int list_get_marker(RGWSI_MetaBackend::Context *ctx, + std::string *marker) override; + + int get_shard_id(RGWSI_MetaBackend::Context *ctx, + const std::string& key, + int *shard_id) override; + + int call(std::optional<RGWSI_MetaBackend_CtxParams> opt, + std::function<int(RGWSI_MetaBackend::Context *)> f) override; +}; + + +class RGWSI_MetaBackend_Handler_SObj : public RGWSI_MetaBackend_Handler { + friend class RGWSI_MetaBackend_SObj::Context_SObj; + + RGWSI_MBSObj_Handler_Module *module{nullptr}; + +public: + RGWSI_MetaBackend_Handler_SObj(RGWSI_MetaBackend *be) : + RGWSI_MetaBackend_Handler(be) {} + + void set_module(RGWSI_MBSObj_Handler_Module *_module) { + module = _module; + } + + RGWSI_MBSObj_Handler_Module *get_module() { + return module; + } +}; diff --git a/src/rgw/services/svc_meta_be_types.h b/src/rgw/services/svc_meta_be_types.h new file mode 100644 index 000000000..4a88a8e0b --- /dev/null +++ b/src/rgw/services/svc_meta_be_types.h @@ -0,0 +1,26 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +enum RGWSI_META_BE_TYPES { + SOBJ = 1, + OTP = 2, + BUCKET = 3, + BI = 4, + USER = 5, +}; + diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc new file mode 100644 index 000000000..43f84ed0a --- /dev/null +++ b/src/rgw/services/svc_notify.cc @@ -0,0 +1,515 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "include/random.h" +#include "include/Context.h" +#include "common/errno.h" + +#include "rgw_cache.h" +#include "svc_notify.h" +#include "svc_finisher.h" +#include "svc_zone.h" +#include "svc_rados.h" + +#include "rgw_zone.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +static string notify_oid_prefix = "notify"; + +RGWSI_Notify::~RGWSI_Notify() +{ + shutdown(); +} + + +class RGWWatcher : public DoutPrefixProvider , public librados::WatchCtx2 { + CephContext *cct; + RGWSI_Notify *svc; + int index; + RGWSI_RADOS::Obj obj; + uint64_t watch_handle; + int register_ret{0}; + bool unregister_done{false}; + librados::AioCompletion *register_completion{nullptr}; + + class C_ReinitWatch : public Context { + RGWWatcher *watcher; + public: + explicit C_ReinitWatch(RGWWatcher *_watcher) : watcher(_watcher) {} + void finish(int r) override { + watcher->reinit(); + } + }; + + CephContext *get_cct() const override { return cct; } + unsigned get_subsys() const override { return dout_subsys; } + std::ostream& gen_prefix(std::ostream& out) const override { + return out << "rgw watcher librados: "; + } + +public: + RGWWatcher(CephContext *_cct, RGWSI_Notify *s, int i, RGWSI_RADOS::Obj& o) : cct(_cct), svc(s), index(i), obj(o), watch_handle(0) {} + void handle_notify(uint64_t notify_id, + uint64_t cookie, + uint64_t notifier_id, + bufferlist& bl) override { + ldpp_dout(this, 10) << "RGWWatcher::handle_notify() " + << " notify_id " << notify_id + << " cookie " << cookie + << " notifier " << notifier_id + << " bl.length()=" << bl.length() << dendl; + + if (unlikely(svc->inject_notify_timeout_probability == 1) || + (svc->inject_notify_timeout_probability > 0 && + (svc->inject_notify_timeout_probability > + ceph::util::generate_random_number(0.0, 1.0)))) { + ldpp_dout(this, 0) + << "RGWWatcher::handle_notify() dropping notification! " + << "If this isn't what you want, set " + << "rgw_inject_notify_timeout_probability to zero!" << dendl; + return; + } + + svc->watch_cb(this, notify_id, cookie, notifier_id, bl); + + bufferlist reply_bl; // empty reply payload + obj.notify_ack(notify_id, cookie, reply_bl); + } + void handle_error(uint64_t cookie, int err) override { + ldpp_dout(this, -1) << "RGWWatcher::handle_error cookie " << cookie + << " err " << cpp_strerror(err) << dendl; + svc->remove_watcher(index); + svc->schedule_context(new C_ReinitWatch(this)); + } + + void reinit() { + if(!unregister_done) { + int ret = unregister_watch(); + if (ret < 0) { + ldout(cct, 0) << "ERROR: unregister_watch() returned ret=" << ret << dendl; + } + } + int ret = register_watch(); + if (ret < 0) { + ldout(cct, 0) << "ERROR: register_watch() returned ret=" << ret << dendl; + svc->schedule_context(new C_ReinitWatch(this)); + return; + } + } + + int unregister_watch() { + int r = svc->unwatch(obj, watch_handle); + unregister_done = true; + if (r < 0) { + return r; + } + svc->remove_watcher(index); + return 0; + } + + int register_watch_async() { + if (register_completion) { + register_completion->release(); + register_completion = nullptr; + } + register_completion = librados::Rados::aio_create_completion(nullptr, nullptr); + register_ret = obj.aio_watch(register_completion, &watch_handle, this); + if (register_ret < 0) { + register_completion->release(); + return register_ret; + } + return 0; + } + + int register_watch_finish() { + if (register_ret < 0) { + return register_ret; + } + if (!register_completion) { + return -EINVAL; + } + register_completion->wait_for_complete(); + int r = register_completion->get_return_value(); + register_completion->release(); + register_completion = nullptr; + if (r < 0) { + return r; + } + svc->add_watcher(index); + unregister_done = false; + return 0; + } + + int register_watch() { + int r = obj.watch(&watch_handle, this); + if (r < 0) { + return r; + } + svc->add_watcher(index); + unregister_done = false; + return 0; + } +}; + + +class RGWSI_Notify_ShutdownCB : public RGWSI_Finisher::ShutdownCB +{ + RGWSI_Notify *svc; +public: + RGWSI_Notify_ShutdownCB(RGWSI_Notify *_svc) : svc(_svc) {} + void call() override { + svc->shutdown(); + } +}; + +string RGWSI_Notify::get_control_oid(int i) +{ + char buf[notify_oid_prefix.size() + 16]; + snprintf(buf, sizeof(buf), "%s.%d", notify_oid_prefix.c_str(), i); + + return string(buf); +} + +// do not call pick_obj_control before init_watch +RGWSI_RADOS::Obj RGWSI_Notify::pick_control_obj(const string& key) +{ + uint32_t r = ceph_str_hash_linux(key.c_str(), key.size()); + + int i = r % num_watchers; + return notify_objs[i]; +} + +int RGWSI_Notify::init_watch(const DoutPrefixProvider *dpp, optional_yield y) +{ + num_watchers = cct->_conf->rgw_num_control_oids; + + bool compat_oid = (num_watchers == 0); + + if (num_watchers <= 0) + num_watchers = 1; + + watchers = new RGWWatcher *[num_watchers]; + + int error = 0; + + notify_objs.resize(num_watchers); + + for (int i=0; i < num_watchers; i++) { + string notify_oid; + + if (!compat_oid) { + notify_oid = get_control_oid(i); + } else { + notify_oid = notify_oid_prefix; + } + + notify_objs[i] = rados_svc->handle().obj({control_pool, notify_oid}); + auto& notify_obj = notify_objs[i]; + + int r = notify_obj.open(dpp); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: notify_obj.open() returned r=" << r << dendl; + return r; + } + + librados::ObjectWriteOperation op; + op.create(false); + r = notify_obj.operate(dpp, &op, y); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "ERROR: notify_obj.operate() returned r=" << r << dendl; + return r; + } + + RGWWatcher *watcher = new RGWWatcher(cct, this, i, notify_obj); + watchers[i] = watcher; + + r = watcher->register_watch_async(); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: register_watch_aio() returned " << r << dendl; + error = r; + continue; + } + } + + for (int i = 0; i < num_watchers; ++i) { + int r = watchers[i]->register_watch_finish(); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: async watch returned " << r << dendl; + error = r; + } + } + + if (error < 0) { + return error; + } + + return 0; +} + +void RGWSI_Notify::finalize_watch() +{ + for (int i = 0; i < num_watchers; i++) { + RGWWatcher *watcher = watchers[i]; + if (watchers_set.find(i) != watchers_set.end()) + watcher->unregister_watch(); + delete watcher; + } + + delete[] watchers; +} + +int RGWSI_Notify::do_start(optional_yield y, const DoutPrefixProvider *dpp) +{ + int r = zone_svc->start(y, dpp); + if (r < 0) { + return r; + } + + assert(zone_svc->is_started()); /* otherwise there's an ordering problem */ + + r = rados_svc->start(y, dpp); + if (r < 0) { + return r; + } + r = finisher_svc->start(y, dpp); + if (r < 0) { + return r; + } + + inject_notify_timeout_probability = + cct->_conf.get_val<double>("rgw_inject_notify_timeout_probability"); + max_notify_retries = cct->_conf.get_val<uint64_t>("rgw_max_notify_retries"); + + control_pool = zone_svc->get_zone_params().control_pool; + + int ret = init_watch(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed to initialize watch: " << cpp_strerror(-ret) << dendl; + return ret; + } + + shutdown_cb = new RGWSI_Notify_ShutdownCB(this); + int handle; + finisher_svc->register_caller(shutdown_cb, &handle); + finisher_handle = handle; + + return 0; +} + +void RGWSI_Notify::shutdown() +{ + if (finalized) { + return; + } + + if (finisher_handle) { + finisher_svc->unregister_caller(*finisher_handle); + } + finalize_watch(); + + delete shutdown_cb; + + finalized = true; +} + +int RGWSI_Notify::unwatch(RGWSI_RADOS::Obj& obj, uint64_t watch_handle) +{ + int r = obj.unwatch(watch_handle); + if (r < 0) { + ldout(cct, 0) << "ERROR: rados->unwatch2() returned r=" << r << dendl; + return r; + } + r = rados_svc->handle().watch_flush(); + if (r < 0) { + ldout(cct, 0) << "ERROR: rados->watch_flush() returned r=" << r << dendl; + return r; + } + return 0; +} + +void RGWSI_Notify::add_watcher(int i) +{ + ldout(cct, 20) << "add_watcher() i=" << i << dendl; + std::unique_lock l{watchers_lock}; + watchers_set.insert(i); + if (watchers_set.size() == (size_t)num_watchers) { + ldout(cct, 2) << "all " << num_watchers << " watchers are set, enabling cache" << dendl; + _set_enabled(true); + } +} + +void RGWSI_Notify::remove_watcher(int i) +{ + ldout(cct, 20) << "remove_watcher() i=" << i << dendl; + std::unique_lock l{watchers_lock}; + size_t orig_size = watchers_set.size(); + watchers_set.erase(i); + if (orig_size == (size_t)num_watchers && + watchers_set.size() < orig_size) { /* actually removed */ + ldout(cct, 2) << "removed watcher, disabling cache" << dendl; + _set_enabled(false); + } +} + +int RGWSI_Notify::watch_cb(const DoutPrefixProvider *dpp, + uint64_t notify_id, + uint64_t cookie, + uint64_t notifier_id, + bufferlist& bl) +{ + std::shared_lock l{watchers_lock}; + if (cb) { + return cb->watch_cb(dpp, notify_id, cookie, notifier_id, bl); + } + return 0; +} + +void RGWSI_Notify::set_enabled(bool status) +{ + std::unique_lock l{watchers_lock}; + _set_enabled(status); +} + +void RGWSI_Notify::_set_enabled(bool status) +{ + enabled = status; + if (cb) { + cb->set_enabled(status); + } +} + +int RGWSI_Notify::distribute(const DoutPrefixProvider *dpp, const string& key, + const RGWCacheNotifyInfo& cni, + optional_yield y) +{ + /* The RGW uses the control pool to store the watch notify objects. + The precedence in RGWSI_Notify::do_start is to call to zone_svc->start and later to init_watch(). + The first time, RGW starts in the cluster, the RGW will try to create zone and zonegroup system object. + In that case RGW will try to distribute the cache before it ran init_watch, + which will lead to division by 0 in pick_obj_control (num_watchers is 0). + */ + if (num_watchers > 0) { + RGWSI_RADOS::Obj notify_obj = pick_control_obj(key); + + ldpp_dout(dpp, 10) << "distributing notification oid=" << notify_obj.get_ref().obj + << " cni=" << cni << dendl; + return robust_notify(dpp, notify_obj, cni, y); + } + return 0; +} + +namespace librados { + +static std::ostream& operator<<(std::ostream& out, const notify_timeout_t& t) +{ + return out << t.notifier_id << ':' << t.cookie; +} + +} // namespace librados + +using timeout_vector = std::vector<librados::notify_timeout_t>; + +static timeout_vector decode_timeouts(const bufferlist& bl) +{ + using ceph::decode; + auto p = bl.begin(); + + // decode and discard the acks + uint32_t num_acks; + decode(num_acks, p); + for (auto i = 0u; i < num_acks; ++i) { + std::pair<uint64_t, uint64_t> id; + decode(id, p); + // discard the payload + uint32_t blen; + decode(blen, p); + p += blen; + } + + // decode and return the timeouts + uint32_t num_timeouts; + decode(num_timeouts, p); + + timeout_vector timeouts; + for (auto i = 0u; i < num_timeouts; ++i) { + std::pair<uint64_t, uint64_t> id; + decode(id, p); + timeouts.push_back({id.first, id.second}); + } + return timeouts; +} + +int RGWSI_Notify::robust_notify(const DoutPrefixProvider *dpp, + RGWSI_RADOS::Obj& notify_obj, + const RGWCacheNotifyInfo& cni, + optional_yield y) +{ + bufferlist bl, rbl; + encode(cni, bl); + + // First, try to send, without being fancy about it. + auto r = notify_obj.notify(dpp, bl, 0, &rbl, y); + + if (r < 0) { + timeout_vector timeouts; + try { + timeouts = decode_timeouts(rbl); + } catch (const buffer::error& e) { + ldpp_dout(dpp, 0) << "robust_notify failed to decode notify response: " + << e.what() << dendl; + } + + ldpp_dout(dpp, 1) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " Watchers " << timeouts << " did not respond." + << " Notify failed on object " << cni.obj << ": " + << cpp_strerror(-r) << dendl; + } + + // If we timed out, get serious. + if (r == -ETIMEDOUT) { + RGWCacheNotifyInfo info; + info.op = INVALIDATE_OBJ; + info.obj = cni.obj; + bufferlist retrybl; + encode(info, retrybl); + + for (auto tries = 0u; + r == -ETIMEDOUT && tries < max_notify_retries; + ++tries) { + ldpp_dout(dpp, 1) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " Invalidating obj=" << info.obj << " tries=" + << tries << dendl; + r = notify_obj.notify(dpp, retrybl, 0, &rbl, y); + if (r < 0) { + timeout_vector timeouts; + try { + timeouts = decode_timeouts(rbl); + } catch (const buffer::error& e) { + ldpp_dout(dpp, 0) << "robust_notify failed to decode notify response: " + << e.what() << dendl; + } + + ldpp_dout(dpp, 1) << __PRETTY_FUNCTION__ << ":" << __LINE__ + << " Watchers " << timeouts << " did not respond." + << " Invalidation attempt " << tries << " failed: " + << cpp_strerror(-r) << dendl; + } + } + } + return r; +} + +void RGWSI_Notify::register_watch_cb(CB *_cb) +{ + std::unique_lock l{watchers_lock}; + cb = _cb; + _set_enabled(enabled); +} + +void RGWSI_Notify::schedule_context(Context *c) +{ + finisher_svc->schedule_context(c); +} diff --git a/src/rgw/services/svc_notify.h b/src/rgw/services/svc_notify.h new file mode 100644 index 000000000..f7329136e --- /dev/null +++ b/src/rgw/services/svc_notify.h @@ -0,0 +1,106 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" + +#include "svc_rados.h" + + +class Context; + +class RGWSI_Zone; +class RGWSI_Finisher; + +class RGWWatcher; +class RGWSI_Notify_ShutdownCB; +struct RGWCacheNotifyInfo; + +class RGWSI_Notify : public RGWServiceInstance +{ + friend class RGWWatcher; + friend class RGWSI_Notify_ShutdownCB; + friend class RGWServices_Def; + +public: + class CB; + +private: + RGWSI_Zone *zone_svc{nullptr}; + RGWSI_RADOS *rados_svc{nullptr}; + RGWSI_Finisher *finisher_svc{nullptr}; + + ceph::shared_mutex watchers_lock = ceph::make_shared_mutex("watchers_lock"); + rgw_pool control_pool; + + int num_watchers{0}; + RGWWatcher **watchers{nullptr}; + std::set<int> watchers_set; + std::vector<RGWSI_RADOS::Obj> notify_objs; + + bool enabled{false}; + + double inject_notify_timeout_probability{0}; + uint64_t max_notify_retries = 10; + + std::string get_control_oid(int i); + RGWSI_RADOS::Obj pick_control_obj(const std::string& key); + + CB *cb{nullptr}; + + std::optional<int> finisher_handle; + RGWSI_Notify_ShutdownCB *shutdown_cb{nullptr}; + + bool finalized{false}; + + int init_watch(const DoutPrefixProvider *dpp, optional_yield y); + void finalize_watch(); + + void init(RGWSI_Zone *_zone_svc, + RGWSI_RADOS *_rados_svc, + RGWSI_Finisher *_finisher_svc) { + zone_svc = _zone_svc; + rados_svc = _rados_svc; + finisher_svc = _finisher_svc; + } + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + void shutdown() override; + + int unwatch(RGWSI_RADOS::Obj& obj, uint64_t watch_handle); + void add_watcher(int i); + void remove_watcher(int i); + + int watch_cb(const DoutPrefixProvider *dpp, + uint64_t notify_id, + uint64_t cookie, + uint64_t notifier_id, + bufferlist& bl); + void _set_enabled(bool status); + void set_enabled(bool status); + + int robust_notify(const DoutPrefixProvider *dpp, RGWSI_RADOS::Obj& notify_obj, + const RGWCacheNotifyInfo& bl, optional_yield y); + + void schedule_context(Context *c); +public: + RGWSI_Notify(CephContext *cct): RGWServiceInstance(cct) {} + + virtual ~RGWSI_Notify() override; + + class CB { + public: + virtual ~CB() {} + virtual int watch_cb(const DoutPrefixProvider *dpp, + uint64_t notify_id, + uint64_t cookie, + uint64_t notifier_id, + bufferlist& bl) = 0; + virtual void set_enabled(bool status) = 0; + }; + + int distribute(const DoutPrefixProvider *dpp, const std::string& key, const RGWCacheNotifyInfo& bl, + optional_yield y); + + void register_watch_cb(CB *cb); +}; diff --git a/src/rgw/services/svc_otp.cc b/src/rgw/services/svc_otp.cc new file mode 100644 index 000000000..81d8d5711 --- /dev/null +++ b/src/rgw/services/svc_otp.cc @@ -0,0 +1,186 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_otp.h" +#include "svc_zone.h" +#include "svc_meta.h" +#include "svc_meta_be_sobj.h" + +#include "rgw_zone.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +class RGW_MB_Handler_Module_OTP : public RGWSI_MBSObj_Handler_Module { + RGWSI_Zone *zone_svc; + string prefix; +public: + RGW_MB_Handler_Module_OTP(RGWSI_Zone *_zone_svc) : RGWSI_MBSObj_Handler_Module("otp"), + zone_svc(_zone_svc) {} + + void get_pool_and_oid(const string& key, rgw_pool *pool, string *oid) override { + if (pool) { + *pool = zone_svc->get_zone_params().otp_pool; + } + + if (oid) { + *oid = key; + } + } + + const string& get_oid_prefix() override { + return prefix; + } + + bool is_valid_oid(const string& oid) override { + return true; + } + + string key_to_oid(const string& key) override { + return key; + } + + string oid_to_key(const string& oid) override { + return oid; + } +}; + +RGWSI_OTP::RGWSI_OTP(CephContext *cct): RGWServiceInstance(cct) { +} + +RGWSI_OTP::~RGWSI_OTP() { +} + +void RGWSI_OTP::init(RGWSI_Zone *_zone_svc, + RGWSI_Meta *_meta_svc, + RGWSI_MetaBackend *_meta_be_svc) +{ + svc.otp = this; + svc.zone = _zone_svc; + svc.meta = _meta_svc; + svc.meta_be = _meta_be_svc; +} + +int RGWSI_OTP::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + /* create first backend handler for bucket entrypoints */ + + RGWSI_MetaBackend_Handler *_otp_be_handler; + + int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_OTP, &_otp_be_handler); + if (r < 0) { + ldout(ctx(), 0) << "ERROR: failed to create be handler: r=" << r << dendl; + return r; + } + + be_handler = _otp_be_handler; + + RGWSI_MetaBackend_Handler_OTP *otp_be_handler = static_cast<RGWSI_MetaBackend_Handler_OTP *>(_otp_be_handler); + + auto otp_be_module = new RGW_MB_Handler_Module_OTP(svc.zone); + be_module.reset(otp_be_module); + otp_be_handler->set_module(otp_be_module); + + return 0; +} + +int RGWSI_OTP::read_all(RGWSI_OTP_BE_Ctx& ctx, + const string& key, + otp_devices_list_t *devices, + real_time *pmtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) +{ + RGWSI_MBOTP_GetParams params; + params.pdevices = devices; + params.pmtime = pmtime; + + int ret = svc.meta_be->get_entry(ctx.get(), key, params, objv_tracker, y, dpp); + if (ret < 0) { + return ret; + } + + return 0; +} + +int RGWSI_OTP::read_all(RGWSI_OTP_BE_Ctx& ctx, + const rgw_user& uid, + otp_devices_list_t *devices, + real_time *pmtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + return read_all(ctx, + uid.to_str(), + devices, + pmtime, + objv_tracker, + y, + dpp); +} + +int RGWSI_OTP::store_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const string& key, + const otp_devices_list_t& devices, + real_time mtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + RGWSI_MBOTP_PutParams params; + params.mtime = mtime; + params.devices = devices; + + int ret = svc.meta_be->put_entry(dpp, ctx.get(), key, params, objv_tracker, y); + if (ret < 0) { + return ret; + } + + return 0; +} + +int RGWSI_OTP::store_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const rgw_user& uid, + const otp_devices_list_t& devices, + real_time mtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + return store_all(dpp, ctx, + uid.to_str(), + devices, + mtime, + objv_tracker, + y); +} + +int RGWSI_OTP::remove_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const string& key, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + RGWSI_MBOTP_RemoveParams params; + + int ret = svc.meta_be->remove_entry(dpp, ctx.get(), key, params, objv_tracker, y); + if (ret < 0) { + return ret; + } + + return 0; +} + +int RGWSI_OTP::remove_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const rgw_user& uid, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + return remove_all(dpp,ctx, + uid.to_str(), + objv_tracker, + y); +} diff --git a/src/rgw/services/svc_otp.h b/src/rgw/services/svc_otp.h new file mode 100644 index 000000000..e639c2c92 --- /dev/null +++ b/src/rgw/services/svc_otp.h @@ -0,0 +1,95 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "cls/otp/cls_otp_types.h" + +#include "rgw_service.h" + +#include "svc_otp_types.h" +#include "svc_meta_be_otp.h" + +class RGWSI_Zone; + +class RGWSI_OTP : public RGWServiceInstance +{ + RGWSI_OTP_BE_Handler be_handler; + std::unique_ptr<RGWSI_MetaBackend::Module> be_module; + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + +public: + struct Svc { + RGWSI_OTP *otp{nullptr}; + RGWSI_Zone *zone{nullptr}; + RGWSI_Meta *meta{nullptr}; + RGWSI_MetaBackend *meta_be{nullptr}; + } svc; + + RGWSI_OTP(CephContext *cct); + ~RGWSI_OTP(); + + RGWSI_OTP_BE_Handler& get_be_handler() { + return be_handler; + } + + void init(RGWSI_Zone *_zone_svc, + RGWSI_Meta *_meta_svc, + RGWSI_MetaBackend *_meta_be_svc); + + int read_all(RGWSI_OTP_BE_Ctx& ctx, + const std::string& key, + otp_devices_list_t *devices, + real_time *pmtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp); + int read_all(RGWSI_OTP_BE_Ctx& ctx, + const rgw_user& uid, + otp_devices_list_t *devices, + real_time *pmtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp); + int store_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const std::string& key, + const otp_devices_list_t& devices, + real_time mtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y); + int store_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const rgw_user& uid, + const otp_devices_list_t& devices, + real_time mtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y); + int remove_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const std::string& key, + RGWObjVersionTracker *objv_tracker, + optional_yield y); + int remove_all(const DoutPrefixProvider *dpp, + RGWSI_OTP_BE_Ctx& ctx, + const rgw_user& uid, + RGWObjVersionTracker *objv_tracker, + optional_yield y); +}; + + diff --git a/src/rgw/services/svc_otp_types.h b/src/rgw/services/svc_otp_types.h new file mode 100644 index 000000000..60e2a79d6 --- /dev/null +++ b/src/rgw/services/svc_otp_types.h @@ -0,0 +1,29 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "common/ptr_wrapper.h" + +#include "svc_meta_be.h" +#include "svc_meta_be_types.h" + +class RGWSI_MetaBackend_Handler; + +using RGWSI_OTP_BE_Handler = ptr_wrapper<RGWSI_MetaBackend_Handler, RGWSI_META_BE_TYPES::OTP>; +using RGWSI_OTP_BE_Ctx = ptr_wrapper<RGWSI_MetaBackend::Context, RGWSI_META_BE_TYPES::OTP>; + diff --git a/src/rgw/services/svc_quota.cc b/src/rgw/services/svc_quota.cc new file mode 100644 index 000000000..3108a1173 --- /dev/null +++ b/src/rgw/services/svc_quota.cc @@ -0,0 +1,18 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_quota.h" +#include "svc_zone.h" + +#include "rgw_zone.h" + +const RGWQuotaInfo& RGWSI_Quota::get_bucket_quota() const +{ + return zone_svc->get_current_period().get_config().quota.bucket_quota; +} + +const RGWQuotaInfo& RGWSI_Quota::get_user_quota() const +{ + return zone_svc->get_current_period().get_config().quota.user_quota; +} + diff --git a/src/rgw/services/svc_quota.h b/src/rgw/services/svc_quota.h new file mode 100644 index 000000000..81aa0e1bd --- /dev/null +++ b/src/rgw/services/svc_quota.h @@ -0,0 +1,22 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" + + +class RGWSI_Quota : public RGWServiceInstance +{ + RGWSI_Zone *zone_svc{nullptr}; + +public: + RGWSI_Quota(CephContext *cct): RGWServiceInstance(cct) {} + + void init(RGWSI_Zone *_zone_svc) { + zone_svc = _zone_svc; + } + + const RGWQuotaInfo& get_bucket_quota() const; + const RGWQuotaInfo& get_user_quota() const; +}; diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc new file mode 100644 index 000000000..99f400f42 --- /dev/null +++ b/src/rgw/services/svc_rados.cc @@ -0,0 +1,445 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_rados.h" + +#include "include/rados/librados.hpp" +#include "common/errno.h" +#include "osd/osd_types.h" +#include "rgw_tools.h" +#include "rgw_cr_rados.h" + +#include "auth/AuthRegistry.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +RGWSI_RADOS::RGWSI_RADOS(CephContext *cct) : RGWServiceInstance(cct) +{ +} + +RGWSI_RADOS::~RGWSI_RADOS() +{ +} + +int RGWSI_RADOS::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + int ret = rados.init_with_context(cct); + if (ret < 0) { + return ret; + } + ret = rados.connect(); + if (ret < 0) { + return ret; + } + + async_processor.reset(new RGWAsyncRadosProcessor(cct, cct->_conf->rgw_num_async_rados_threads)); + async_processor->start(); + + return 0; +} + +void RGWSI_RADOS::shutdown() +{ + if (async_processor) { + async_processor->stop(); + } + rados.shutdown(); +} + +void RGWSI_RADOS::stop_processor() +{ + if (async_processor) { + async_processor->stop(); + } +} + +librados::Rados* RGWSI_RADOS::get_rados_handle() +{ + return &rados; +} + +std::string RGWSI_RADOS::cluster_fsid() +{ + std::string fsid; + (void) get_rados_handle()->cluster_fsid(&fsid); + return fsid; +} + +uint64_t RGWSI_RADOS::instance_id() +{ + return get_rados_handle()->get_instance_id(); +} + +int RGWSI_RADOS::open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, + const OpenParams& params) +{ + return rgw_init_ioctx(dpp, get_rados_handle(), pool, io_ctx, + params.create, + params.mostly_omap); +} + +int RGWSI_RADOS::pool_iterate(const DoutPrefixProvider *dpp, + librados::IoCtx& io_ctx, + librados::NObjectIterator& iter, + uint32_t num, vector<rgw_bucket_dir_entry>& objs, + RGWAccessListFilter *filter, + bool *is_truncated) +{ + if (iter == io_ctx.nobjects_end()) + return -ENOENT; + + uint32_t i; + + for (i = 0; i < num && iter != io_ctx.nobjects_end(); ++i, ++iter) { + rgw_bucket_dir_entry e; + + string oid = iter->get_oid(); + ldpp_dout(dpp, 20) << "RGWRados::pool_iterate: got " << oid << dendl; + + // fill it in with initial values; we may correct later + if (filter && !filter->filter(oid, oid)) + continue; + + e.key = oid; + objs.push_back(e); + } + + if (is_truncated) + *is_truncated = (iter != io_ctx.nobjects_end()); + + return objs.size(); +} + +RGWSI_RADOS::Obj::Obj(Pool& pool, const string& oid) : rados_svc(pool.rados_svc) +{ + ref.pool = pool; + ref.obj = rgw_raw_obj(pool.get_pool(), oid); +} + +void RGWSI_RADOS::Obj::init(const rgw_raw_obj& obj) +{ + ref.pool = RGWSI_RADOS::Pool(rados_svc, obj.pool); + ref.obj = obj; +} + +int RGWSI_RADOS::Obj::open(const DoutPrefixProvider *dpp) +{ + int r = ref.pool.open(dpp); + if (r < 0) { + return r; + } + + ref.pool.ioctx().locator_set_key(ref.obj.loc); + + return 0; +} + +int RGWSI_RADOS::Obj::operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op, + optional_yield y, int flags) +{ + return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, y, flags); +} + +int RGWSI_RADOS::Obj::operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op, + bufferlist *pbl, optional_yield y, int flags) +{ + return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, pbl, y, flags); +} + +int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op) +{ + return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op); +} + +int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op, + bufferlist *pbl) +{ + return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op, pbl); +} + +int RGWSI_RADOS::Obj::watch(uint64_t *handle, librados::WatchCtx2 *ctx) +{ + return ref.pool.ioctx().watch2(ref.obj.oid, handle, ctx); +} + +int RGWSI_RADOS::Obj::aio_watch(librados::AioCompletion *c, uint64_t *handle, librados::WatchCtx2 *ctx) +{ + return ref.pool.ioctx().aio_watch(ref.obj.oid, c, handle, ctx); +} + +int RGWSI_RADOS::Obj::unwatch(uint64_t handle) +{ + return ref.pool.ioctx().unwatch2(handle); +} + +int RGWSI_RADOS::Obj::notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms, + bufferlist *pbl, optional_yield y) +{ + return rgw_rados_notify(dpp, ref.pool.ioctx(), ref.obj.oid, bl, timeout_ms, pbl, y); +} + +void RGWSI_RADOS::Obj::notify_ack(uint64_t notify_id, + uint64_t cookie, + bufferlist& bl) +{ + ref.pool.ioctx().notify_ack(ref.obj.oid, notify_id, cookie, bl); +} + +uint64_t RGWSI_RADOS::Obj::get_last_version() +{ + return ref.pool.ioctx().get_last_version(); +} + +int RGWSI_RADOS::Pool::create(const DoutPrefixProvider *dpp) +{ + librados::Rados *rad = rados_svc->get_rados_handle(); + int r = rad->pool_create(pool.name.c_str()); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: pool_create returned " << r << dendl; + return r; + } + librados::IoCtx io_ctx; + r = rad->ioctx_create(pool.name.c_str(), io_ctx); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: ioctx_create returned " << r << dendl; + return r; + } + r = io_ctx.application_enable(pg_pool_t::APPLICATION_NAME_RGW, false); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: application_enable returned " << r << dendl; + return r; + } + return 0; +} + +int RGWSI_RADOS::Pool::create(const DoutPrefixProvider *dpp, const vector<rgw_pool>& pools, vector<int> *retcodes) +{ + vector<librados::PoolAsyncCompletion *> completions; + vector<int> rets; + + librados::Rados *rad = rados_svc->get_rados_handle(); + for (auto iter = pools.begin(); iter != pools.end(); ++iter) { + librados::PoolAsyncCompletion *c = librados::Rados::pool_async_create_completion(); + completions.push_back(c); + auto& pool = *iter; + int ret = rad->pool_create_async(pool.name.c_str(), c); + rets.push_back(ret); + } + + vector<int>::iterator riter; + vector<librados::PoolAsyncCompletion *>::iterator citer; + + bool error = false; + ceph_assert(rets.size() == completions.size()); + for (riter = rets.begin(), citer = completions.begin(); riter != rets.end(); ++riter, ++citer) { + int r = *riter; + librados::PoolAsyncCompletion *c = *citer; + if (r == 0) { + c->wait(); + r = c->get_return_value(); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: async pool_create returned " << r << dendl; + error = true; + } + } + c->release(); + retcodes->push_back(r); + } + if (error) { + return 0; + } + + std::vector<librados::IoCtx> io_ctxs; + retcodes->clear(); + for (auto pool : pools) { + io_ctxs.emplace_back(); + int ret = rad->ioctx_create(pool.name.c_str(), io_ctxs.back()); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: ioctx_create returned " << ret << dendl; + error = true; + } + retcodes->push_back(ret); + } + if (error) { + return 0; + } + + completions.clear(); + for (auto &io_ctx : io_ctxs) { + librados::PoolAsyncCompletion *c = + librados::Rados::pool_async_create_completion(); + completions.push_back(c); + int ret = io_ctx.application_enable_async(pg_pool_t::APPLICATION_NAME_RGW, + false, c); + ceph_assert(ret == 0); + } + + retcodes->clear(); + for (auto c : completions) { + c->wait(); + int ret = c->get_return_value(); + if (ret == -EOPNOTSUPP) { + ret = 0; + } else if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: async application_enable returned " << ret + << dendl; + error = true; + } + c->release(); + retcodes->push_back(ret); + } + return 0; +} + +int RGWSI_RADOS::Pool::lookup() +{ + librados::Rados *rad = rados_svc->get_rados_handle(); + int ret = rad->pool_lookup(pool.name.c_str()); + if (ret < 0) { + return ret; + } + + return 0; +} + +int RGWSI_RADOS::Pool::open(const DoutPrefixProvider *dpp, const OpenParams& params) +{ + return rados_svc->open_pool_ctx(dpp, pool, state.ioctx, params); +} + +int RGWSI_RADOS::Pool::List::init(const DoutPrefixProvider *dpp, const string& marker, RGWAccessListFilter *filter) +{ + if (ctx.initialized) { + return -EINVAL; + } + + if (!pool) { + return -EINVAL; + } + + int r = pool->rados_svc->open_pool_ctx(dpp, pool->pool, ctx.ioctx); + if (r < 0) { + return r; + } + + librados::ObjectCursor oc; + if (!oc.from_str(marker)) { + ldpp_dout(dpp, 10) << "failed to parse cursor: " << marker << dendl; + return -EINVAL; + } + + try { + ctx.iter = ctx.ioctx.nobjects_begin(oc); + ctx.filter = filter; + ctx.initialized = true; + return 0; + } catch (const std::system_error& e) { + r = -e.code().value(); + ldpp_dout(dpp, 10) << "nobjects_begin threw " << e.what() + << ", returning " << r << dendl; + return r; + } catch (const std::exception& e) { + ldpp_dout(dpp, 10) << "nobjects_begin threw " << e.what() + << ", returning -5" << dendl; + return -EIO; + } +} + +int RGWSI_RADOS::Pool::List::get_next(const DoutPrefixProvider *dpp, + int max, + std::vector<string> *oids, + bool *is_truncated) +{ + if (!ctx.initialized) { + return -EINVAL; + } + vector<rgw_bucket_dir_entry> objs; + int r = pool->rados_svc->pool_iterate(dpp, ctx.ioctx, ctx.iter, max, objs, ctx.filter, is_truncated); + if (r < 0) { + if(r != -ENOENT) { + ldpp_dout(dpp, 10) << "failed to list objects pool_iterate returned r=" << r << dendl; + } + return r; + } + + for (auto& o : objs) { + oids->push_back(o.key.name); + } + + return oids->size(); +} + +RGWSI_RADOS::Obj RGWSI_RADOS::Handle::obj(const rgw_raw_obj& o) +{ + return RGWSI_RADOS::Obj(rados_svc, o); +} +int RGWSI_RADOS::Handle::watch_flush() +{ + librados::Rados *rad = rados_svc->get_rados_handle(); + return rad->watch_flush(); +} + +int RGWSI_RADOS::Handle::mon_command(std::string cmd, + const bufferlist& inbl, + bufferlist *outbl, + std::string *outs) +{ + librados::Rados *rad = rados_svc->get_rados_handle(); + return rad->mon_command(cmd, inbl, outbl, outs); +} + +int RGWSI_RADOS::Pool::List::get_marker(string *marker) +{ + if (!ctx.initialized) { + return -EINVAL; + } + + *marker = ctx.iter.get_cursor().to_str(); + return 0; +} + +int RGWSI_RADOS::clog_warn(const string& msg) +{ + string cmd = + "{" + "\"prefix\": \"log\", " + "\"level\": \"warn\", " + "\"logtext\": [\"" + msg + "\"]" + "}"; + + bufferlist inbl; + auto h = handle(); + return h.mon_command(cmd, inbl, nullptr, nullptr); +} + +bool RGWSI_RADOS::check_secure_mon_conn(const DoutPrefixProvider *dpp) const +{ + AuthRegistry reg(cct); + + reg.refresh_config(); + + std::vector<uint32_t> methods; + std::vector<uint32_t> modes; + + reg.get_supported_methods(CEPH_ENTITY_TYPE_MON, &methods, &modes); + ldpp_dout(dpp, 20) << __func__ << "(): auth registy supported: methods=" << methods << " modes=" << modes << dendl; + + for (auto method : methods) { + if (!reg.is_secure_method(method)) { + ldpp_dout(dpp, 20) << __func__ << "(): method " << method << " is insecure" << dendl; + return false; + } + } + + for (auto mode : modes) { + if (!reg.is_secure_mode(mode)) { + ldpp_dout(dpp, 20) << __func__ << "(): mode " << mode << " is insecure" << dendl; + return false; + } + } + + return true; +} + diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h new file mode 100644 index 000000000..ede029aa8 --- /dev/null +++ b/src/rgw/services/svc_rados.h @@ -0,0 +1,252 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" + +#include "include/rados/librados.hpp" +#include "common/async/yield_context.h" + +class RGWAsyncRadosProcessor; + +class RGWAccessListFilter { +public: + virtual ~RGWAccessListFilter() {} + virtual bool filter(const std::string& name, std::string& key) = 0; +}; + +struct RGWAccessListFilterPrefix : public RGWAccessListFilter { + std::string prefix; + + explicit RGWAccessListFilterPrefix(const std::string& _prefix) : prefix(_prefix) {} + bool filter(const std::string& name, std::string& key) override { + return (prefix.compare(key.substr(0, prefix.size())) == 0); + } +}; + +class RGWSI_RADOS : public RGWServiceInstance +{ + librados::Rados rados; + std::unique_ptr<RGWAsyncRadosProcessor> async_processor; + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + +public: + struct OpenParams { + bool create{true}; + bool mostly_omap{false}; + + OpenParams() {} + + OpenParams& set_create(bool _create) { + create = _create; + return *this; + } + OpenParams& set_mostly_omap(bool _mostly_omap) { + mostly_omap = _mostly_omap; + return *this; + } + }; + +private: + int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, + const OpenParams& params = {}); + int pool_iterate(const DoutPrefixProvider *dpp, + librados::IoCtx& ioctx, + librados::NObjectIterator& iter, + uint32_t num, std::vector<rgw_bucket_dir_entry>& objs, + RGWAccessListFilter *filter, + bool *is_truncated); + +public: + RGWSI_RADOS(CephContext *cct); + ~RGWSI_RADOS(); + librados::Rados* get_rados_handle(); + + void init() {} + void shutdown() override; + void stop_processor(); + + std::string cluster_fsid(); + uint64_t instance_id(); + bool check_secure_mon_conn(const DoutPrefixProvider *dpp) const; + + RGWAsyncRadosProcessor *get_async_processor() { + return async_processor.get(); + } + + int clog_warn(const std::string& msg); + + class Handle; + + class Pool { + friend class RGWSI_RADOS; + friend Handle; + friend class Obj; + + RGWSI_RADOS *rados_svc{nullptr}; + rgw_pool pool; + + struct State { + librados::IoCtx ioctx; + } state; + + Pool(RGWSI_RADOS *_rados_svc, + const rgw_pool& _pool) : rados_svc(_rados_svc), + pool(_pool) {} + + Pool(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {} + public: + Pool() {} + + int create(const DoutPrefixProvider *dpp); + int create(const DoutPrefixProvider *dpp, const std::vector<rgw_pool>& pools, std::vector<int> *retcodes); + int lookup(); + int open(const DoutPrefixProvider *dpp, const OpenParams& params = {}); + + const rgw_pool& get_pool() { + return pool; + } + + librados::IoCtx& ioctx() & { + return state.ioctx; + } + + librados::IoCtx&& ioctx() && { + return std::move(state.ioctx); + } + + struct List { + Pool *pool{nullptr}; + + struct Ctx { + bool initialized{false}; + librados::IoCtx ioctx; + librados::NObjectIterator iter; + RGWAccessListFilter *filter{nullptr}; + } ctx; + + List() {} + List(Pool *_pool) : pool(_pool) {} + + int init(const DoutPrefixProvider *dpp, const std::string& marker, RGWAccessListFilter *filter = nullptr); + int get_next(const DoutPrefixProvider *dpp, int max, + std::vector<std::string> *oids, + bool *is_truncated); + + int get_marker(std::string *marker); + }; + + List op() { + return List(this); + } + + friend List; + }; + + + struct rados_ref { + RGWSI_RADOS::Pool pool; + rgw_raw_obj obj; + }; + + class Obj { + friend class RGWSI_RADOS; + friend class Handle; + + RGWSI_RADOS *rados_svc{nullptr}; + rados_ref ref; + + void init(const rgw_raw_obj& obj); + + Obj(RGWSI_RADOS *_rados_svc, const rgw_raw_obj& _obj) + : rados_svc(_rados_svc) { + init(_obj); + } + + Obj(Pool& pool, const std::string& oid); + + public: + Obj() {} + + int open(const DoutPrefixProvider *dpp); + + int operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op, optional_yield y, + int flags = 0); + int operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op, bufferlist *pbl, + optional_yield y, int flags = 0); + int aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op); + int aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op, + bufferlist *pbl); + + int watch(uint64_t *handle, librados::WatchCtx2 *ctx); + int aio_watch(librados::AioCompletion *c, uint64_t *handle, librados::WatchCtx2 *ctx); + int unwatch(uint64_t handle); + int notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms, + bufferlist *pbl, optional_yield y); + void notify_ack(uint64_t notify_id, + uint64_t cookie, + bufferlist& bl); + + uint64_t get_last_version(); + + rados_ref& get_ref() { return ref; } + const rados_ref& get_ref() const { return ref; } + + const rgw_raw_obj& get_raw_obj() const { + return ref.obj; + } + }; + + class Handle { + friend class RGWSI_RADOS; + + RGWSI_RADOS *rados_svc{nullptr}; + + Handle(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {} + public: + Obj obj(const rgw_raw_obj& o); + + Pool pool(const rgw_pool& p) { + return Pool(rados_svc, p); + } + + int watch_flush(); + + int mon_command(std::string cmd, + const bufferlist& inbl, + bufferlist *outbl, + std::string *outs); + }; + + Handle handle() { + return Handle(this); + } + + Obj obj(const rgw_raw_obj& o) { + return Obj(this, o); + } + + Obj obj(Pool& pool, const std::string& oid) { + return Obj(pool, oid); + } + + Pool pool() { + return Pool(this); + } + + Pool pool(const rgw_pool& p) { + return Pool(this, p); + } + + friend Obj; + friend Pool; + friend Pool::List; +}; + +using rgw_rados_ref = RGWSI_RADOS::rados_ref; + +inline std::ostream& operator<<(std::ostream& out, const RGWSI_RADOS::Obj& obj) { + return out << obj.get_raw_obj(); +} diff --git a/src/rgw/services/svc_role_rados.cc b/src/rgw/services/svc_role_rados.cc new file mode 100644 index 000000000..a84022497 --- /dev/null +++ b/src/rgw/services/svc_role_rados.cc @@ -0,0 +1,82 @@ +#include "svc_role_rados.h" +#include "svc_meta_be_sobj.h" +#include "svc_meta.h" +#include "rgw_role.h" +#include "rgw_zone.h" +#include "svc_zone.h" +#include "rgw_tools.h" + +#define dout_subsys ceph_subsys_rgw + +class RGWSI_Role_Module : public RGWSI_MBSObj_Handler_Module { + RGWSI_Role_RADOS::Svc& svc; + const std::string prefix; +public: + RGWSI_Role_Module(RGWSI_Role_RADOS::Svc& _svc): RGWSI_MBSObj_Handler_Module("roles"), + svc(_svc), + prefix(role_oid_prefix) {} + + void get_pool_and_oid(const std::string& key, + rgw_pool *pool, + std::string *oid) override + { + if (pool) { + *pool = svc.zone->get_zone_params().roles_pool; + } + + if (oid) { + *oid = key_to_oid(key); + } + } + + bool is_valid_oid(const std::string& oid) override { + return boost::algorithm::starts_with(oid, prefix); + } + + std::string key_to_oid(const std::string& key) override { + return prefix + key; + } + + // This is called after `is_valid_oid` and is assumed to be a valid oid + std::string oid_to_key(const std::string& oid) override { + return oid.substr(prefix.size()); + } + + const std::string& get_oid_prefix() { + return prefix; + } +}; + +RGWSI_MetaBackend_Handler* RGWSI_Role_RADOS::get_be_handler() +{ + return be_handler; +} + +void RGWSI_Role_RADOS::init(RGWSI_Zone *_zone_svc, + RGWSI_Meta *_meta_svc, + RGWSI_MetaBackend *_meta_be_svc, + RGWSI_SysObj *_sysobj_svc) +{ + svc.zone = _zone_svc; + svc.meta = _meta_svc; + svc.meta_be = _meta_be_svc; + svc.sysobj = _sysobj_svc; +} + +int RGWSI_Role_RADOS::do_start(optional_yield y, const DoutPrefixProvider *dpp) +{ + + int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, + &be_handler); + if (r < 0) { + ldout(ctx(), 0) << "ERROR: failed to create be_handler for Roles: r=" + << r <<dendl; + return r; + } + + auto module = new RGWSI_Role_Module(svc); + RGWSI_MetaBackend_Handler_SObj* bh= static_cast<RGWSI_MetaBackend_Handler_SObj *>(be_handler); + be_module.reset(module); + bh->set_module(module); + return 0; +} diff --git a/src/rgw/services/svc_role_rados.h b/src/rgw/services/svc_role_rados.h new file mode 100644 index 000000000..d4d3530c2 --- /dev/null +++ b/src/rgw/services/svc_role_rados.h @@ -0,0 +1,50 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 SUSE LLC + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include "rgw_service.h" +#include "rgw_role.h" +#include "svc_meta_be.h" + +class RGWSI_Role_RADOS: public RGWServiceInstance +{ + public: + struct Svc { + RGWSI_Zone *zone{nullptr}; + RGWSI_Meta *meta{nullptr}; + RGWSI_MetaBackend *meta_be{nullptr}; + RGWSI_SysObj *sysobj{nullptr}; + } svc; + + RGWSI_Role_RADOS(CephContext *cct) : RGWServiceInstance(cct) {} + ~RGWSI_Role_RADOS() {} + + void init(RGWSI_Zone *_zone_svc, + RGWSI_Meta *_meta_svc, + RGWSI_MetaBackend *_meta_be_svc, + RGWSI_SysObj *_sysobj_svc); + + RGWSI_MetaBackend_Handler * get_be_handler(); + int do_start(optional_yield y, const DoutPrefixProvider *dpp) override; + +private: + RGWSI_MetaBackend_Handler *be_handler; + std::unique_ptr<RGWSI_MetaBackend::Module> be_module; +}; + +static const std::string role_name_oid_prefix = "role_names."; +static const std::string role_oid_prefix = "roles."; +static const std::string role_path_oid_prefix = "role_paths."; diff --git a/src/rgw/services/svc_sync_modules.cc b/src/rgw/services/svc_sync_modules.cc new file mode 100644 index 000000000..ba9e7d172 --- /dev/null +++ b/src/rgw/services/svc_sync_modules.cc @@ -0,0 +1,44 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_sync_modules.h" +#include "svc_zone.h" + +#include "rgw_sync_module.h" +#include "rgw_zone.h" + +#define dout_subsys ceph_subsys_rgw + +void RGWSI_SyncModules::init(RGWSI_Zone *zone_svc) +{ + svc.zone = zone_svc; + sync_modules_manager = new RGWSyncModulesManager(); + rgw_register_sync_modules(sync_modules_manager); +} + +int RGWSI_SyncModules::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + auto& zone_public_config = svc.zone->get_zone(); + + int ret = sync_modules_manager->create_instance(dpp, cct, zone_public_config.tier_type, svc.zone->get_zone_params().tier_config, &sync_module); + if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed to start sync module instance, ret=" << ret << dendl; + if (ret == -ENOENT) { + ldpp_dout(dpp, -1) << "ERROR: " << zone_public_config.tier_type + << " sync module does not exist. valid sync modules: " + << sync_modules_manager->get_registered_module_names() + << dendl; + } + return ret; + } + + ldpp_dout(dpp, 20) << "started sync module instance, tier type = " << zone_public_config.tier_type << dendl; + + return 0; +} + +RGWSI_SyncModules::~RGWSI_SyncModules() +{ + delete sync_modules_manager; +} + diff --git a/src/rgw/services/svc_sync_modules.h b/src/rgw/services/svc_sync_modules.h new file mode 100644 index 000000000..ea78f5817 --- /dev/null +++ b/src/rgw/services/svc_sync_modules.h @@ -0,0 +1,34 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" +#include "rgw_sync_module.h" + +class RGWSI_Zone; + +class RGWSyncModulesManager; + +class RGWSI_SyncModules : public RGWServiceInstance +{ + RGWSyncModulesManager *sync_modules_manager{nullptr}; + RGWSyncModuleInstanceRef sync_module; + + struct Svc { + RGWSI_Zone *zone{nullptr}; + } svc; + +public: + RGWSI_SyncModules(CephContext *cct): RGWServiceInstance(cct) {} + ~RGWSI_SyncModules(); + + RGWSyncModulesManager *get_manager() { + return sync_modules_manager; + } + + void init(RGWSI_Zone *zone_svc); + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + + RGWSyncModuleInstanceRef& get_sync_module() { return sync_module; } +}; diff --git a/src/rgw/services/svc_sys_obj.cc b/src/rgw/services/svc_sys_obj.cc new file mode 100644 index 000000000..310e60514 --- /dev/null +++ b/src/rgw/services/svc_sys_obj.cc @@ -0,0 +1,183 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_sys_obj.h" +#include "svc_sys_obj_core.h" +#include "svc_rados.h" +#include "svc_zone.h" + +#include "rgw_zone.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +RGWSI_SysObj::Obj RGWSI_SysObj::get_obj(const rgw_raw_obj& obj) +{ + return Obj(core_svc, obj); +} + +RGWSI_SysObj::Obj::ROp::ROp(Obj& _source) : source(_source) { + state.emplace<RGWSI_SysObj_Core::GetObjState>(); +} + +int RGWSI_SysObj::Obj::ROp::stat(optional_yield y, const DoutPrefixProvider *dpp) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.obj; + + return svc->stat(*state, obj, attrs, raw_attrs, + lastmod, obj_size, objv_tracker, y, dpp); +} + +int RGWSI_SysObj::Obj::ROp::read(const DoutPrefixProvider *dpp, + int64_t ofs, int64_t end, bufferlist *bl, + optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.get_obj(); + + return svc->read(dpp, *state, + objv_tracker, + obj, bl, ofs, end, + lastmod, obj_size, + attrs, + raw_attrs, + cache_info, + refresh_version, y); +} + +int RGWSI_SysObj::Obj::ROp::get_attr(const DoutPrefixProvider *dpp, + const char *name, bufferlist *dest, + optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.get_obj(); + + return svc->get_attr(dpp, obj, name, dest, y); +} + +int RGWSI_SysObj::Obj::WOp::remove(const DoutPrefixProvider *dpp, optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.get_obj(); + + return svc->remove(dpp, objv_tracker, obj, y); +} + +int RGWSI_SysObj::Obj::WOp::write(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.get_obj(); + + return svc->write(dpp, obj, pmtime, attrs, exclusive, + bl, objv_tracker, mtime, y); +} + +int RGWSI_SysObj::Obj::WOp::write_data(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.get_obj(); + + return svc->write_data(dpp, obj, bl, exclusive, objv_tracker, y); +} + +int RGWSI_SysObj::Obj::WOp::write_attrs(const DoutPrefixProvider *dpp, optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.get_obj(); + + return svc->set_attrs(dpp, obj, attrs, nullptr, objv_tracker, exclusive, y); +} + +int RGWSI_SysObj::Obj::WOp::write_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& bl, + optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.get_obj(); + + map<string, bufferlist> m; + m[name] = bl; + + return svc->set_attrs(dpp, obj, m, nullptr, objv_tracker, exclusive, y); +} + +int RGWSI_SysObj::Pool::list_prefixed_objs(const DoutPrefixProvider *dpp, const string& prefix, std::function<void(const string&)> cb) +{ + return core_svc->pool_list_prefixed_objs(dpp, pool, prefix, cb); +} + +int RGWSI_SysObj::Pool::Op::init(const DoutPrefixProvider *dpp, const string& marker, const string& prefix) +{ + return source.core_svc->pool_list_objects_init(dpp, source.pool, marker, prefix, &ctx); +} + +int RGWSI_SysObj::Pool::Op::get_next(const DoutPrefixProvider *dpp, int max, vector<string> *oids, bool *is_truncated) +{ + return source.core_svc->pool_list_objects_next(dpp, ctx, max, oids, is_truncated); +} + +int RGWSI_SysObj::Pool::Op::get_marker(string *marker) +{ + return source.core_svc->pool_list_objects_get_marker(ctx, marker); +} + +int RGWSI_SysObj::Obj::OmapOp::get_all(const DoutPrefixProvider *dpp, std::map<string, bufferlist> *m, + optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.obj; + + return svc->omap_get_all(dpp, obj, m, y); +} + +int RGWSI_SysObj::Obj::OmapOp::get_vals(const DoutPrefixProvider *dpp, + const string& marker, uint64_t count, + std::map<string, bufferlist> *m, + bool *pmore, optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.obj; + + return svc->omap_get_vals(dpp, obj, marker, count, m, pmore, y); +} + +int RGWSI_SysObj::Obj::OmapOp::set(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& bl, + optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.obj; + + return svc->omap_set(dpp, obj, key, bl, must_exist, y); +} + +int RGWSI_SysObj::Obj::OmapOp::set(const DoutPrefixProvider *dpp, const map<std::string, bufferlist>& m, + optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.obj; + + return svc->omap_set(dpp, obj, m, must_exist, y); +} + +int RGWSI_SysObj::Obj::OmapOp::del(const DoutPrefixProvider *dpp, const std::string& key, optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.obj; + + return svc->omap_del(dpp, obj, key, y); +} + +int RGWSI_SysObj::Obj::WNOp::notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms, + bufferlist *pbl, optional_yield y) +{ + RGWSI_SysObj_Core *svc = source.core_svc; + rgw_raw_obj& obj = source.obj; + + return svc->notify(dpp, obj, bl, timeout_ms, pbl, y); +} + +RGWSI_Zone *RGWSI_SysObj::get_zone_svc() +{ + return core_svc->get_zone_svc(); +} diff --git a/src/rgw/services/svc_sys_obj.h b/src/rgw/services/svc_sys_obj.h new file mode 100644 index 000000000..f3e217dbd --- /dev/null +++ b/src/rgw/services/svc_sys_obj.h @@ -0,0 +1,270 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "common/static_ptr.h" + +#include "rgw_service.h" + +#include "svc_rados.h" +#include "svc_sys_obj_types.h" +#include "svc_sys_obj_core_types.h" + + +class RGWSI_Zone; +class RGWSI_SysObj; + +struct rgw_cache_entry_info; + +class RGWSI_SysObj : public RGWServiceInstance +{ + friend struct RGWServices_Def; + +public: + class Obj { + friend class ROp; + + RGWSI_SysObj_Core *core_svc; + rgw_raw_obj obj; + + public: + Obj(RGWSI_SysObj_Core *_core_svc, const rgw_raw_obj& _obj) + : core_svc(_core_svc), obj(_obj) {} + + rgw_raw_obj& get_obj() { + return obj; + } + + struct ROp { + Obj& source; + + ceph::static_ptr<RGWSI_SysObj_Obj_GetObjState, sizeof(RGWSI_SysObj_Core_GetObjState)> state; + + RGWObjVersionTracker *objv_tracker{nullptr}; + std::map<std::string, bufferlist> *attrs{nullptr}; + bool raw_attrs{false}; + boost::optional<obj_version> refresh_version{boost::none}; + ceph::real_time *lastmod{nullptr}; + uint64_t *obj_size{nullptr}; + rgw_cache_entry_info *cache_info{nullptr}; + + ROp& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) { + objv_tracker = _objv_tracker; + return *this; + } + + ROp& set_last_mod(ceph::real_time *_lastmod) { + lastmod = _lastmod; + return *this; + } + + ROp& set_obj_size(uint64_t *_obj_size) { + obj_size = _obj_size; + return *this; + } + + ROp& set_attrs(std::map<std::string, bufferlist> *_attrs) { + attrs = _attrs; + return *this; + } + + ROp& set_raw_attrs(bool ra) { + raw_attrs = ra; + return *this; + } + + ROp& set_refresh_version(boost::optional<obj_version>& rf) { + refresh_version = rf; + return *this; + } + + ROp& set_cache_info(rgw_cache_entry_info *ci) { + cache_info = ci; + return *this; + } + + ROp(Obj& _source); + + int stat(optional_yield y, const DoutPrefixProvider *dpp); + int read(const DoutPrefixProvider *dpp, int64_t ofs, int64_t end, bufferlist *pbl, optional_yield y); + int read(const DoutPrefixProvider *dpp, bufferlist *pbl, optional_yield y) { + return read(dpp, 0, -1, pbl, y); + } + int get_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist *dest, optional_yield y); + }; + + struct WOp { + Obj& source; + + RGWObjVersionTracker *objv_tracker{nullptr}; + std::map<std::string, bufferlist> attrs; + ceph::real_time mtime; + ceph::real_time *pmtime{nullptr}; + bool exclusive{false}; + + WOp& set_objv_tracker(RGWObjVersionTracker *_objv_tracker) { + objv_tracker = _objv_tracker; + return *this; + } + + WOp& set_attrs(std::map<std::string, bufferlist>& _attrs) { + attrs = _attrs; + return *this; + } + + WOp& set_attrs(std::map<std::string, bufferlist>&& _attrs) { + attrs = _attrs; + return *this; + } + + WOp& set_mtime(const ceph::real_time& _mtime) { + mtime = _mtime; + return *this; + } + + WOp& set_pmtime(ceph::real_time *_pmtime) { + pmtime = _pmtime; + return *this; + } + + WOp& set_exclusive(bool _exclusive = true) { + exclusive = _exclusive; + return *this; + } + + WOp(Obj& _source) : source(_source) {} + + int remove(const DoutPrefixProvider *dpp, optional_yield y); + int write(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y); + + int write_data(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y); /* write data only */ + int write_attrs(const DoutPrefixProvider *dpp, optional_yield y); /* write attrs only */ + int write_attr(const DoutPrefixProvider *dpp, const char *name, bufferlist& bl, + optional_yield y); /* write attrs only */ + }; + + struct OmapOp { + Obj& source; + + bool must_exist{false}; + + OmapOp& set_must_exist(bool _must_exist = true) { + must_exist = _must_exist; + return *this; + } + + OmapOp(Obj& _source) : source(_source) {} + + int get_all(const DoutPrefixProvider *dpp, std::map<std::string, bufferlist> *m, optional_yield y); + int get_vals(const DoutPrefixProvider *dpp, const std::string& marker, uint64_t count, + std::map<std::string, bufferlist> *m, + bool *pmore, optional_yield y); + int set(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& bl, optional_yield y); + int set(const DoutPrefixProvider *dpp, const std::map<std::string, bufferlist>& m, optional_yield y); + int del(const DoutPrefixProvider *dpp, const std::string& key, optional_yield y); + }; + + struct WNOp { + Obj& source; + + WNOp(Obj& _source) : source(_source) {} + + int notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl, + optional_yield y); + }; + ROp rop() { + return ROp(*this); + } + + WOp wop() { + return WOp(*this); + } + + OmapOp omap() { + return OmapOp(*this); + } + + WNOp wn() { + return WNOp(*this); + } + }; + + class Pool { + friend class Op; + friend class RGWSI_SysObj_Core; + + RGWSI_SysObj_Core *core_svc; + rgw_pool pool; + + protected: + using ListImplInfo = RGWSI_SysObj_Pool_ListInfo; + + struct ListCtx { + ceph::static_ptr<ListImplInfo, sizeof(RGWSI_SysObj_Core_PoolListImplInfo)> impl; /* update this if creating new backend types */ + }; + + public: + Pool(RGWSI_SysObj_Core *_core_svc, + const rgw_pool& _pool) : core_svc(_core_svc), + pool(_pool) {} + + rgw_pool& get_pool() { + return pool; + } + + struct Op { + Pool& source; + ListCtx ctx; + + Op(Pool& _source) : source(_source) {} + + int init(const DoutPrefixProvider *dpp, const std::string& marker, const std::string& prefix); + int get_next(const DoutPrefixProvider *dpp, int max, std::vector<std::string> *oids, bool *is_truncated); + int get_marker(std::string *marker); + }; + + int list_prefixed_objs(const DoutPrefixProvider *dpp, const std::string& prefix, std::function<void(const std::string&)> cb); + + template <typename Container> + int list_prefixed_objs(const DoutPrefixProvider *dpp, const std::string& prefix, + Container *result) { + return list_prefixed_objs(dpp, prefix, [&](const std::string& val) { + result->push_back(val); + }); + } + + Op op() { + return Op(*this); + } + }; + + friend class Obj; + friend class Obj::ROp; + friend class Obj::WOp; + friend class Pool; + friend class Pool::Op; + +protected: + RGWSI_RADOS *rados_svc{nullptr}; + RGWSI_SysObj_Core *core_svc{nullptr}; + + void init(RGWSI_RADOS *_rados_svc, + RGWSI_SysObj_Core *_core_svc) { + rados_svc = _rados_svc; + core_svc = _core_svc; + } + +public: + RGWSI_SysObj(CephContext *cct): RGWServiceInstance(cct) {} + + Obj get_obj(const rgw_raw_obj& obj); + + Pool get_pool(const rgw_pool& pool) { + return Pool(core_svc, pool); + } + + RGWSI_Zone *get_zone_svc(); +}; + +using RGWSysObj = RGWSI_SysObj::Obj; diff --git a/src/rgw/services/svc_sys_obj_cache.cc b/src/rgw/services/svc_sys_obj_cache.cc new file mode 100644 index 000000000..d1b7a3dbb --- /dev/null +++ b/src/rgw/services/svc_sys_obj_cache.cc @@ -0,0 +1,670 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "common/admin_socket.h" + +#include "svc_sys_obj_cache.h" +#include "svc_zone.h" +#include "svc_notify.h" + +#include "rgw_zone.h" +#include "rgw_tools.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +class RGWSI_SysObj_Cache_CB : public RGWSI_Notify::CB +{ + RGWSI_SysObj_Cache *svc; +public: + RGWSI_SysObj_Cache_CB(RGWSI_SysObj_Cache *_svc) : svc(_svc) {} + int watch_cb(const DoutPrefixProvider *dpp, + uint64_t notify_id, + uint64_t cookie, + uint64_t notifier_id, + bufferlist& bl) { + return svc->watch_cb(dpp, notify_id, cookie, notifier_id, bl); + } + + void set_enabled(bool status) { + svc->set_enabled(status); + } +}; + +int RGWSI_SysObj_Cache::do_start(optional_yield y, const DoutPrefixProvider *dpp) +{ + int r = asocket.start(); + if (r < 0) { + return r; + } + + r = RGWSI_SysObj_Core::do_start(y, dpp); + if (r < 0) { + return r; + } + + r = notify_svc->start(y, dpp); + if (r < 0) { + return r; + } + + assert(notify_svc->is_started()); + + cb.reset(new RGWSI_SysObj_Cache_CB(this)); + + notify_svc->register_watch_cb(cb.get()); + + return 0; +} + +void RGWSI_SysObj_Cache::shutdown() +{ + asocket.shutdown(); + RGWSI_SysObj_Core::shutdown(); +} + +static string normal_name(rgw_pool& pool, const std::string& oid) { + std::string buf; + buf.reserve(pool.name.size() + pool.ns.size() + oid.size() + 2); + buf.append(pool.name).append("+").append(pool.ns).append("+").append(oid); + return buf; +} + +void RGWSI_SysObj_Cache::normalize_pool_and_obj(const rgw_pool& src_pool, const string& src_obj, rgw_pool& dst_pool, string& dst_obj) +{ + if (src_obj.size()) { + dst_pool = src_pool; + dst_obj = src_obj; + } else { + dst_pool = zone_svc->get_zone_params().domain_root; + dst_obj = src_pool.name; + } +} + + +int RGWSI_SysObj_Cache::remove(const DoutPrefixProvider *dpp, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + optional_yield y) + +{ + rgw_pool pool; + string oid; + normalize_pool_and_obj(obj.pool, obj.oid, pool, oid); + + string name = normal_name(pool, oid); + cache.invalidate_remove(dpp, name); + + ObjectCacheInfo info; + int r = distribute_cache(dpp, name, obj, info, INVALIDATE_OBJ, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to distribute cache: r=" << r << dendl; + } + + return RGWSI_SysObj_Core::remove(dpp, objv_tracker, obj, y); +} + +int RGWSI_SysObj_Cache::read(const DoutPrefixProvider *dpp, + RGWSI_SysObj_Obj_GetObjState& read_state, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + bufferlist *obl, off_t ofs, off_t end, + ceph::real_time* pmtime, uint64_t* psize, + map<string, bufferlist> *attrs, + bool raw_attrs, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version> refresh_version, + optional_yield y) +{ + rgw_pool pool; + string oid; + if (ofs != 0) { + return RGWSI_SysObj_Core::read(dpp, read_state, objv_tracker, obj, obl, + ofs, end, pmtime, psize, attrs, raw_attrs, + cache_info, refresh_version, y); + } + + normalize_pool_and_obj(obj.pool, obj.oid, pool, oid); + string name = normal_name(pool, oid); + + ObjectCacheInfo info; + + uint32_t flags = (end != 0 ? CACHE_FLAG_DATA : 0); + if (objv_tracker) + flags |= CACHE_FLAG_OBJV; + if (pmtime || psize) + flags |= CACHE_FLAG_META; + if (attrs) + flags |= CACHE_FLAG_XATTRS; + + int r = cache.get(dpp, name, info, flags, cache_info); + if (r == 0 && + (!refresh_version || !info.version.compare(&(*refresh_version)))) { + if (info.status < 0) + return info.status; + + bufferlist& bl = info.data; + + bufferlist::iterator i = bl.begin(); + + obl->clear(); + + i.copy_all(*obl); + if (objv_tracker) + objv_tracker->read_version = info.version; + if (pmtime) { + *pmtime = info.meta.mtime; + } + if (psize) { + *psize = info.meta.size; + } + if (attrs) { + if (raw_attrs) { + *attrs = info.xattrs; + } else { + rgw_filter_attrset(info.xattrs, RGW_ATTR_PREFIX, attrs); + } + } + return obl->length(); + } + if(r == -ENODATA) + return -ENOENT; + + // if we only ask for one of mtime or size, ask for the other too so we can + // satisfy CACHE_FLAG_META + uint64_t size = 0; + real_time mtime; + if (pmtime) { + if (!psize) { + psize = &size; + } + } else if (psize) { + if (!pmtime) { + pmtime = &mtime; + } + } + + map<string, bufferlist> unfiltered_attrset; + r = RGWSI_SysObj_Core::read(dpp, read_state, objv_tracker, + obj, obl, ofs, end, pmtime, psize, + (attrs ? &unfiltered_attrset : nullptr), + true, /* cache unfiltered attrs */ + cache_info, + refresh_version, y); + if (r < 0) { + if (r == -ENOENT) { // only update ENOENT, we'd rather retry other errors + info.status = r; + cache.put(dpp, name, info, cache_info); + } + return r; + } + + if (obl->length() == end + 1) { + /* in this case, most likely object contains more data, we can't cache it */ + flags &= ~CACHE_FLAG_DATA; + } else { + bufferptr p(r); + bufferlist& bl = info.data; + bl.clear(); + bufferlist::iterator o = obl->begin(); + o.copy_all(bl); + } + + info.status = 0; + info.flags = flags; + if (objv_tracker) { + info.version = objv_tracker->read_version; + } + if (pmtime) { + info.meta.mtime = *pmtime; + } + if (psize) { + info.meta.size = *psize; + } + if (attrs) { + info.xattrs = std::move(unfiltered_attrset); + if (raw_attrs) { + *attrs = info.xattrs; + } else { + rgw_filter_attrset(info.xattrs, RGW_ATTR_PREFIX, attrs); + } + } + cache.put(dpp, name, info, cache_info); + return r; +} + +int RGWSI_SysObj_Cache::get_attr(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const char *attr_name, + bufferlist *dest, + optional_yield y) +{ + rgw_pool pool; + string oid; + + normalize_pool_and_obj(obj.pool, obj.oid, pool, oid); + string name = normal_name(pool, oid); + + ObjectCacheInfo info; + + uint32_t flags = CACHE_FLAG_XATTRS; + + int r = cache.get(dpp, name, info, flags, nullptr); + if (r == 0) { + if (info.status < 0) + return info.status; + + auto iter = info.xattrs.find(attr_name); + if (iter == info.xattrs.end()) { + return -ENODATA; + } + + *dest = iter->second; + return dest->length(); + } else if (r == -ENODATA) { + return -ENOENT; + } + /* don't try to cache this one */ + return RGWSI_SysObj_Core::get_attr(dpp, obj, attr_name, dest, y); +} + +int RGWSI_SysObj_Cache::set_attrs(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + map<string, bufferlist>& attrs, + map<string, bufferlist> *rmattrs, + RGWObjVersionTracker *objv_tracker, + bool exclusive, optional_yield y) +{ + rgw_pool pool; + string oid; + normalize_pool_and_obj(obj.pool, obj.oid, pool, oid); + ObjectCacheInfo info; + info.xattrs = attrs; + if (rmattrs) { + info.rm_xattrs = *rmattrs; + } + info.status = 0; + info.flags = CACHE_FLAG_MODIFY_XATTRS; + int ret = RGWSI_SysObj_Core::set_attrs(dpp, obj, attrs, rmattrs, objv_tracker, exclusive, y); + string name = normal_name(pool, oid); + if (ret >= 0) { + if (objv_tracker && objv_tracker->read_version.ver) { + info.version = objv_tracker->read_version; + info.flags |= CACHE_FLAG_OBJV; + } + cache.put(dpp, name, info, NULL); + int r = distribute_cache(dpp, name, obj, info, UPDATE_OBJ, y); + if (r < 0) + ldpp_dout(dpp, 0) << "ERROR: failed to distribute cache for " << obj << dendl; + } else { + cache.invalidate_remove(dpp, name); + } + + return ret; +} + +int RGWSI_SysObj_Cache::write(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + real_time *pmtime, + map<std::string, bufferlist>& attrs, + bool exclusive, + const bufferlist& data, + RGWObjVersionTracker *objv_tracker, + real_time set_mtime, + optional_yield y) +{ + rgw_pool pool; + string oid; + normalize_pool_and_obj(obj.pool, obj.oid, pool, oid); + ObjectCacheInfo info; + info.xattrs = attrs; + info.status = 0; + info.data = data; + info.flags = CACHE_FLAG_XATTRS | CACHE_FLAG_DATA | CACHE_FLAG_META; + ceph::real_time result_mtime; + int ret = RGWSI_SysObj_Core::write(dpp, obj, &result_mtime, attrs, + exclusive, data, + objv_tracker, set_mtime, y); + if (pmtime) { + *pmtime = result_mtime; + } + if (objv_tracker && objv_tracker->read_version.ver) { + info.version = objv_tracker->read_version; + info.flags |= CACHE_FLAG_OBJV; + } + info.meta.mtime = result_mtime; + info.meta.size = data.length(); + string name = normal_name(pool, oid); + if (ret >= 0) { + cache.put(dpp, name, info, NULL); + int r = distribute_cache(dpp, name, obj, info, UPDATE_OBJ, y); + if (r < 0) + ldpp_dout(dpp, 0) << "ERROR: failed to distribute cache for " << obj << dendl; + } else { + cache.invalidate_remove(dpp, name); + } + + return ret; +} + +int RGWSI_SysObj_Cache::write_data(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const bufferlist& data, + bool exclusive, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + rgw_pool pool; + string oid; + normalize_pool_and_obj(obj.pool, obj.oid, pool, oid); + + ObjectCacheInfo info; + info.data = data; + info.meta.size = data.length(); + info.status = 0; + info.flags = CACHE_FLAG_DATA; + + int ret = RGWSI_SysObj_Core::write_data(dpp, obj, data, exclusive, objv_tracker, y); + string name = normal_name(pool, oid); + if (ret >= 0) { + if (objv_tracker && objv_tracker->read_version.ver) { + info.version = objv_tracker->read_version; + info.flags |= CACHE_FLAG_OBJV; + } + cache.put(dpp, name, info, NULL); + int r = distribute_cache(dpp, name, obj, info, UPDATE_OBJ, y); + if (r < 0) + ldpp_dout(dpp, 0) << "ERROR: failed to distribute cache for " << obj << dendl; + } else { + cache.invalidate_remove(dpp, name); + } + + return ret; +} + +int RGWSI_SysObj_Cache::raw_stat(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + uint64_t *psize, real_time *pmtime, + map<string, bufferlist> *attrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + rgw_pool pool; + string oid; + normalize_pool_and_obj(obj.pool, obj.oid, pool, oid); + + string name = normal_name(pool, oid); + + uint64_t size; + real_time mtime; + + ObjectCacheInfo info; + uint32_t flags = CACHE_FLAG_META | CACHE_FLAG_XATTRS; + if (objv_tracker) + flags |= CACHE_FLAG_OBJV; + int r = cache.get(dpp, name, info, flags, NULL); + if (r == 0) { + if (info.status < 0) + return info.status; + + size = info.meta.size; + mtime = info.meta.mtime; + if (objv_tracker) + objv_tracker->read_version = info.version; + goto done; + } + if (r == -ENODATA) { + return -ENOENT; + } + r = RGWSI_SysObj_Core::raw_stat(dpp, obj, &size, &mtime, &info.xattrs, + objv_tracker, y); + if (r < 0) { + if (r == -ENOENT) { + info.status = r; + cache.put(dpp, name, info, NULL); + } + return r; + } + info.status = 0; + info.meta.mtime = mtime; + info.meta.size = size; + info.flags = CACHE_FLAG_META | CACHE_FLAG_XATTRS; + if (objv_tracker) { + info.flags |= CACHE_FLAG_OBJV; + info.version = objv_tracker->read_version; + } + cache.put(dpp, name, info, NULL); +done: + if (psize) + *psize = size; + if (pmtime) + *pmtime = mtime; + if (attrs) + *attrs = info.xattrs; + return 0; +} + +int RGWSI_SysObj_Cache::distribute_cache(const DoutPrefixProvider *dpp, + const string& normal_name, + const rgw_raw_obj& obj, + ObjectCacheInfo& obj_info, int op, + optional_yield y) +{ + RGWCacheNotifyInfo info; + info.op = op; + info.obj_info = obj_info; + info.obj = obj; + return notify_svc->distribute(dpp, normal_name, info, y); +} + +int RGWSI_SysObj_Cache::watch_cb(const DoutPrefixProvider *dpp, + uint64_t notify_id, + uint64_t cookie, + uint64_t notifier_id, + bufferlist& bl) +{ + RGWCacheNotifyInfo info; + + try { + auto iter = bl.cbegin(); + decode(info, iter); + } catch (buffer::end_of_buffer& err) { + ldpp_dout(dpp, 0) << "ERROR: got bad notification" << dendl; + return -EIO; + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: buffer::error" << dendl; + return -EIO; + } + + rgw_pool pool; + string oid; + normalize_pool_and_obj(info.obj.pool, info.obj.oid, pool, oid); + string name = normal_name(pool, oid); + + switch (info.op) { + case UPDATE_OBJ: + cache.put(dpp, name, info.obj_info, NULL); + break; + case INVALIDATE_OBJ: + cache.invalidate_remove(dpp, name); + break; + default: + ldpp_dout(dpp, 0) << "WARNING: got unknown notification op: " << info.op << dendl; + return -EINVAL; + } + + return 0; +} + +void RGWSI_SysObj_Cache::set_enabled(bool status) +{ + cache.set_enabled(status); +} + +bool RGWSI_SysObj_Cache::chain_cache_entry(const DoutPrefixProvider *dpp, + std::initializer_list<rgw_cache_entry_info *> cache_info_entries, + RGWChainedCache::Entry *chained_entry) +{ + return cache.chain_cache_entry(dpp, cache_info_entries, chained_entry); +} + +void RGWSI_SysObj_Cache::register_chained_cache(RGWChainedCache *cc) +{ + cache.chain_cache(cc); +} + +void RGWSI_SysObj_Cache::unregister_chained_cache(RGWChainedCache *cc) +{ + cache.unchain_cache(cc); +} + +static void cache_list_dump_helper(Formatter* f, + const std::string& name, + const ceph::real_time mtime, + const std::uint64_t size) +{ + f->dump_string("name", name); + f->dump_string("mtime", ceph::to_iso_8601(mtime)); + f->dump_unsigned("size", size); +} + +class RGWSI_SysObj_Cache_ASocketHook : public AdminSocketHook { + RGWSI_SysObj_Cache *svc; + + static constexpr std::string_view admin_commands[][2] = { + { "cache list name=filter,type=CephString,req=false", + "cache list [filter_str]: list object cache, possibly matching substrings" }, + { "cache inspect name=target,type=CephString,req=true", + "cache inspect target: print cache element" }, + { "cache erase name=target,type=CephString,req=true", + "cache erase target: erase element from cache" }, + { "cache zap", + "cache zap: erase all elements from cache" } + }; + +public: + RGWSI_SysObj_Cache_ASocketHook(RGWSI_SysObj_Cache *_svc) : svc(_svc) {} + + int start(); + void shutdown(); + + int call(std::string_view command, const cmdmap_t& cmdmap, + const bufferlist&, + Formatter *f, + std::ostream& ss, + bufferlist& out) override; +}; + +int RGWSI_SysObj_Cache_ASocketHook::start() +{ + auto admin_socket = svc->ctx()->get_admin_socket(); + for (auto cmd : admin_commands) { + int r = admin_socket->register_command(cmd[0], this, cmd[1]); + if (r < 0) { + ldout(svc->ctx(), 0) << "ERROR: fail to register admin socket command (r=" << r + << ")" << dendl; + return r; + } + } + return 0; +} + +void RGWSI_SysObj_Cache_ASocketHook::shutdown() +{ + auto admin_socket = svc->ctx()->get_admin_socket(); + admin_socket->unregister_commands(this); +} + +int RGWSI_SysObj_Cache_ASocketHook::call( + std::string_view command, const cmdmap_t& cmdmap, + const bufferlist&, + Formatter *f, + std::ostream& ss, + bufferlist& out) +{ + if (command == "cache list"sv) { + std::optional<std::string> filter; + if (auto i = cmdmap.find("filter"); i != cmdmap.cend()) { + filter = boost::get<std::string>(i->second); + } + f->open_array_section("cache_entries"); + svc->asocket.call_list(filter, f); + f->close_section(); + return 0; + } else if (command == "cache inspect"sv) { + const auto& target = boost::get<std::string>(cmdmap.at("target")); + if (svc->asocket.call_inspect(target, f)) { + return 0; + } else { + ss << "Unable to find entry "s + target + ".\n"; + return -ENOENT; + } + } else if (command == "cache erase"sv) { + const auto& target = boost::get<std::string>(cmdmap.at("target")); + if (svc->asocket.call_erase(target)) { + return 0; + } else { + ss << "Unable to find entry "s + target + ".\n"; + return -ENOENT; + } + } else if (command == "cache zap"sv) { + svc->asocket.call_zap(); + return 0; + } + return -ENOSYS; +} + +RGWSI_SysObj_Cache::ASocketHandler::ASocketHandler(const DoutPrefixProvider *_dpp, RGWSI_SysObj_Cache *_svc) : dpp(_dpp), svc(_svc) +{ + hook.reset(new RGWSI_SysObj_Cache_ASocketHook(_svc)); +} + +RGWSI_SysObj_Cache::ASocketHandler::~ASocketHandler() +{ +} + +int RGWSI_SysObj_Cache::ASocketHandler::start() +{ + return hook->start(); +} + +void RGWSI_SysObj_Cache::ASocketHandler::shutdown() +{ + return hook->shutdown(); +} + +void RGWSI_SysObj_Cache::ASocketHandler::call_list(const std::optional<std::string>& filter, Formatter* f) +{ + svc->cache.for_each( + [&filter, f] (const string& name, const ObjectCacheEntry& entry) { + if (!filter || name.find(*filter) != name.npos) { + cache_list_dump_helper(f, name, entry.info.meta.mtime, + entry.info.meta.size); + } + }); +} + +int RGWSI_SysObj_Cache::ASocketHandler::call_inspect(const std::string& target, Formatter* f) +{ + if (const auto entry = svc->cache.get(dpp, target)) { + f->open_object_section("cache_entry"); + f->dump_string("name", target.c_str()); + entry->dump(f); + f->close_section(); + return true; + } else { + return false; + } +} + +int RGWSI_SysObj_Cache::ASocketHandler::call_erase(const std::string& target) +{ + return svc->cache.invalidate_remove(dpp, target); +} + +int RGWSI_SysObj_Cache::ASocketHandler::call_zap() +{ + svc->cache.invalidate_all(); + return 0; +} diff --git a/src/rgw/services/svc_sys_obj_cache.h b/src/rgw/services/svc_sys_obj_cache.h new file mode 100644 index 000000000..f7950843f --- /dev/null +++ b/src/rgw/services/svc_sys_obj_cache.h @@ -0,0 +1,222 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "common/RWLock.h" +#include "rgw_service.h" +#include "rgw_cache.h" + +#include "svc_sys_obj_core.h" + +class RGWSI_Notify; + +class RGWSI_SysObj_Cache_CB; +class RGWSI_SysObj_Cache_ASocketHook; + +class RGWSI_SysObj_Cache : public RGWSI_SysObj_Core +{ + friend class RGWSI_SysObj_Cache_CB; + friend class RGWServices_Def; + friend class ASocketHandler; + + RGWSI_Notify *notify_svc{nullptr}; + ObjectCache cache; + + std::shared_ptr<RGWSI_SysObj_Cache_CB> cb; + + void normalize_pool_and_obj(const rgw_pool& src_pool, const std::string& src_obj, rgw_pool& dst_pool, std::string& dst_obj); +protected: + void init(RGWSI_RADOS *_rados_svc, + RGWSI_Zone *_zone_svc, + RGWSI_Notify *_notify_svc) { + core_init(_rados_svc, _zone_svc); + notify_svc = _notify_svc; + } + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + void shutdown() override; + + int raw_stat(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + uint64_t *psize, real_time *pmtime, + std::map<std::string, bufferlist> *attrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y) override; + + int read(const DoutPrefixProvider *dpp, + RGWSI_SysObj_Obj_GetObjState& read_state, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + bufferlist *bl, off_t ofs, off_t end, + ceph::real_time* pmtime, uint64_t* psize, + std::map<std::string, bufferlist> *attrs, + bool raw_attrs, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version>, + optional_yield y) override; + + int get_attr(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, const char *name, bufferlist *dest, + optional_yield y) override; + + int set_attrs(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + std::map<std::string, bufferlist>& attrs, + std::map<std::string, bufferlist> *rmattrs, + RGWObjVersionTracker *objv_tracker, + bool exclusive, optional_yield y) override; + + int remove(const DoutPrefixProvider *dpp, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + optional_yield y) override; + + int write(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + real_time *pmtime, + std::map<std::string, bufferlist>& attrs, + bool exclusive, + const bufferlist& data, + RGWObjVersionTracker *objv_tracker, + real_time set_mtime, + optional_yield y) override; + + int write_data(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const bufferlist& bl, + bool exclusive, + RGWObjVersionTracker *objv_tracker, + optional_yield y); + + int distribute_cache(const DoutPrefixProvider *dpp, const std::string& normal_name, const rgw_raw_obj& obj, + ObjectCacheInfo& obj_info, int op, + optional_yield y); + + int watch_cb(const DoutPrefixProvider *dpp, + uint64_t notify_id, + uint64_t cookie, + uint64_t notifier_id, + bufferlist& bl); + + void set_enabled(bool status); + +public: + RGWSI_SysObj_Cache(const DoutPrefixProvider *dpp, CephContext *cct) : RGWSI_SysObj_Core(cct), asocket(dpp, this) { + cache.set_ctx(cct); + } + + bool chain_cache_entry(const DoutPrefixProvider *dpp, + std::initializer_list<rgw_cache_entry_info *> cache_info_entries, + RGWChainedCache::Entry *chained_entry); + void register_chained_cache(RGWChainedCache *cc); + void unregister_chained_cache(RGWChainedCache *cc); + + class ASocketHandler { + const DoutPrefixProvider *dpp; + RGWSI_SysObj_Cache *svc; + + std::unique_ptr<RGWSI_SysObj_Cache_ASocketHook> hook; + + public: + ASocketHandler(const DoutPrefixProvider *dpp, RGWSI_SysObj_Cache *_svc); + ~ASocketHandler(); + + int start(); + void shutdown(); + + // `call_list` must iterate over all cache entries and call + // `cache_list_dump_helper` with the supplied Formatter on any that + // include `filter` as a substd::string. + // + void call_list(const std::optional<std::string>& filter, Formatter* f); + + // `call_inspect` must look up the requested target and, if found, + // dump it to the supplied Formatter and return true. If not found, + // it must return false. + // + int call_inspect(const std::string& target, Formatter* f); + + // `call_erase` must erase the requested target and return true. If + // the requested target does not exist, it should return false. + int call_erase(const std::string& target); + + // `call_zap` must erase the cache. + int call_zap(); + } asocket; +}; + +template <class T> +class RGWChainedCacheImpl : public RGWChainedCache { + RGWSI_SysObj_Cache *svc{nullptr}; + ceph::timespan expiry; + RWLock lock; + + std::unordered_map<std::string, std::pair<T, ceph::coarse_mono_time>> entries; + +public: + RGWChainedCacheImpl() : lock("RGWChainedCacheImpl::lock") {} + ~RGWChainedCacheImpl() { + if (!svc) { + return; + } + svc->unregister_chained_cache(this); + } + + void unregistered() override { + svc = nullptr; + } + + void init(RGWSI_SysObj_Cache *_svc) { + if (!_svc) { + return; + } + svc = _svc; + svc->register_chained_cache(this); + expiry = std::chrono::seconds(svc->ctx()->_conf.get_val<uint64_t>( + "rgw_cache_expiry_interval")); + } + + boost::optional<T> find(const std::string& key) { + std::shared_lock rl{lock}; + auto iter = entries.find(key); + if (iter == entries.end()) { + return boost::none; + } + if (expiry.count() && + (ceph::coarse_mono_clock::now() - iter->second.second) > expiry) { + return boost::none; + } + + return iter->second.first; + } + + bool put(const DoutPrefixProvider *dpp, RGWSI_SysObj_Cache *svc, const std::string& key, T *entry, + std::initializer_list<rgw_cache_entry_info *> cache_info_entries) { + if (!svc) { + return false; + } + + Entry chain_entry(this, key, entry); + + /* we need the svc cache to call us under its lock to maintain lock ordering */ + return svc->chain_cache_entry(dpp, cache_info_entries, &chain_entry); + } + + void chain_cb(const std::string& key, void *data) override { + T *entry = static_cast<T *>(data); + std::unique_lock wl{lock}; + entries[key].first = *entry; + if (expiry.count() > 0) { + entries[key].second = ceph::coarse_mono_clock::now(); + } + } + + void invalidate(const std::string& key) override { + std::unique_lock wl{lock}; + entries.erase(key); + } + + void invalidate_all() override { + std::unique_lock wl{lock}; + entries.clear(); + } +}; /* RGWChainedCacheImpl */ diff --git a/src/rgw/services/svc_sys_obj_core.cc b/src/rgw/services/svc_sys_obj_core.cc new file mode 100644 index 000000000..303089691 --- /dev/null +++ b/src/rgw/services/svc_sys_obj_core.cc @@ -0,0 +1,666 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_sys_obj_core.h" +#include "svc_rados.h" +#include "svc_zone.h" + +#include "rgw_tools.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +int RGWSI_SysObj_Core_GetObjState::get_rados_obj(const DoutPrefixProvider *dpp, + RGWSI_RADOS *rados_svc, + RGWSI_Zone *zone_svc, + const rgw_raw_obj& obj, + RGWSI_RADOS::Obj **pobj) +{ + if (!has_rados_obj) { + if (obj.oid.empty()) { + ldpp_dout(dpp, 0) << "ERROR: obj.oid is empty" << dendl; + return -EINVAL; + } + + rados_obj = rados_svc->obj(obj); + int r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + has_rados_obj = true; + } + *pobj = &rados_obj; + return 0; +} + +int RGWSI_SysObj_Core::get_rados_obj(const DoutPrefixProvider *dpp, + RGWSI_Zone *zone_svc, + const rgw_raw_obj& obj, + RGWSI_RADOS::Obj *pobj) +{ + if (obj.oid.empty()) { + ldpp_dout(dpp, 0) << "ERROR: obj.oid is empty" << dendl; + return -EINVAL; + } + + *pobj = rados_svc->obj(obj); + int r = pobj->open(dpp); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_SysObj_Core::raw_stat(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + uint64_t *psize, real_time *pmtime, + map<string, bufferlist> *attrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + return r; + } + + uint64_t size = 0; + struct timespec mtime_ts; + + librados::ObjectReadOperation op; + if (objv_tracker) { + objv_tracker->prepare_op_for_read(&op); + } + op.getxattrs(attrs, nullptr); + if (psize || pmtime) { + op.stat2(&size, &mtime_ts, nullptr); + } + bufferlist outbl; + r = rados_obj.operate(dpp, &op, &outbl, y); + if (r < 0) + return r; + + if (psize) + *psize = size; + if (pmtime) + *pmtime = ceph::real_clock::from_timespec(mtime_ts); + + return 0; +} + +int RGWSI_SysObj_Core::stat(RGWSI_SysObj_Obj_GetObjState& _state, + const rgw_raw_obj& obj, + map<string, bufferlist> *attrs, + bool raw_attrs, + real_time *lastmod, + uint64_t *obj_size, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + uint64_t size = 0; + ceph::real_time mtime; + std::map<std::string, bufferlist> attrset; + + int r = raw_stat(dpp, obj, &size, &mtime, &attrset, objv_tracker, y); + if (r < 0) + return r; + + if (attrs) { + if (raw_attrs) { + *attrs = std::move(attrset); + } else { + rgw_filter_attrset(attrset, RGW_ATTR_PREFIX, attrs); + } + if (cct->_conf->subsys.should_gather<ceph_subsys_rgw, 20>()) { + map<string, bufferlist>::iterator iter; + for (iter = attrs->begin(); iter != attrs->end(); ++iter) { + ldpp_dout(dpp, 20) << "Read xattr: " << iter->first << dendl; + } + } + } + + if (obj_size) + *obj_size = size; + if (lastmod) + *lastmod = mtime; + + return 0; +} + +int RGWSI_SysObj_Core::read(const DoutPrefixProvider *dpp, + RGWSI_SysObj_Obj_GetObjState& _read_state, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + bufferlist *bl, off_t ofs, off_t end, + ceph::real_time* pmtime, uint64_t* psize, + map<string, bufferlist> *attrs, + bool raw_attrs, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version>, + optional_yield y) +{ + auto& read_state = static_cast<GetObjState&>(_read_state); + + uint64_t len; + struct timespec mtime_ts; + librados::ObjectReadOperation op; + + if (end < 0) + len = 0; + else + len = end - ofs + 1; + + if (objv_tracker) { + objv_tracker->prepare_op_for_read(&op); + } + if (psize || pmtime) { + op.stat2(psize, &mtime_ts, nullptr); + } + + ldpp_dout(dpp, 20) << "rados->read ofs=" << ofs << " len=" << len << dendl; + op.read(ofs, len, bl, nullptr); + + map<string, bufferlist> unfiltered_attrset; + + if (attrs) { + if (raw_attrs) { + op.getxattrs(attrs, nullptr); + } else { + op.getxattrs(&unfiltered_attrset, nullptr); + } + } + + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + r = rados_obj.operate(dpp, &op, nullptr, y); + if (r < 0) { + ldpp_dout(dpp, 20) << "rados_obj.operate() r=" << r << " bl.length=" << bl->length() << dendl; + return r; + } + ldpp_dout(dpp, 20) << "rados_obj.operate() r=" << r << " bl.length=" << bl->length() << dendl; + + uint64_t op_ver = rados_obj.get_last_version(); + + if (read_state.last_ver > 0 && + read_state.last_ver != op_ver) { + ldpp_dout(dpp, 5) << "raced with an object write, abort" << dendl; + return -ECANCELED; + } + + if (pmtime) { + *pmtime = ceph::real_clock::from_timespec(mtime_ts); + } + if (attrs && !raw_attrs) { + rgw_filter_attrset(unfiltered_attrset, RGW_ATTR_PREFIX, attrs); + } + + read_state.last_ver = op_ver; + + return bl->length(); +} + +/** + * Get an attribute for a system object. + * obj: the object to get attr + * name: name of the attr to retrieve + * dest: bufferlist to store the result in + * Returns: 0 on success, -ERR# otherwise. + */ +int RGWSI_SysObj_Core::get_attr(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const char *name, + bufferlist *dest, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + librados::ObjectReadOperation op; + + int rval; + op.getxattr(name, dest, &rval); + + r = rados_obj.operate(dpp, &op, nullptr, y); + if (r < 0) + return r; + + return 0; +} + +int RGWSI_SysObj_Core::set_attrs(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + map<string, bufferlist>& attrs, + map<string, bufferlist> *rmattrs, + RGWObjVersionTracker *objv_tracker, + bool exclusive, optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + librados::ObjectWriteOperation op; + + if (exclusive) { + op.create(true); // exclusive create + } + if (objv_tracker) { + objv_tracker->prepare_op_for_write(&op); + } + + map<string, bufferlist>::iterator iter; + if (rmattrs) { + for (iter = rmattrs->begin(); iter != rmattrs->end(); ++iter) { + const string& name = iter->first; + op.rmxattr(name.c_str()); + } + } + + for (iter = attrs.begin(); iter != attrs.end(); ++iter) { + const string& name = iter->first; + bufferlist& bl = iter->second; + + if (!bl.length()) + continue; + + op.setxattr(name.c_str(), bl); + } + + if (!op.size()) + return 0; + + bufferlist bl; + + r = rados_obj.operate(dpp, &op, y); + if (r < 0) + return r; + + if (objv_tracker) { + objv_tracker->apply_write(); + } + return 0; +} + +int RGWSI_SysObj_Core::omap_get_vals(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const string& marker, + uint64_t count, + std::map<string, bufferlist> *m, + bool *pmore, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + string start_after = marker; + bool more; + + do { + librados::ObjectReadOperation op; + + std::map<string, bufferlist> t; + int rval; + op.omap_get_vals2(start_after, count, &t, &more, &rval); + + r = rados_obj.operate(dpp, &op, nullptr, y); + if (r < 0) { + return r; + } + if (t.empty()) { + break; + } + count -= t.size(); + start_after = t.rbegin()->first; + m->insert(t.begin(), t.end()); + } while (more && count > 0); + + if (pmore) { + *pmore = more; + } + return 0; +} + +int RGWSI_SysObj_Core::omap_get_all(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + std::map<string, bufferlist> *m, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + +#define MAX_OMAP_GET_ENTRIES 1024 + const int count = MAX_OMAP_GET_ENTRIES; + string start_after; + bool more; + + do { + librados::ObjectReadOperation op; + + std::map<string, bufferlist> t; + int rval; + op.omap_get_vals2(start_after, count, &t, &more, &rval); + + r = rados_obj.operate(dpp, &op, nullptr, y); + if (r < 0) { + return r; + } + if (t.empty()) { + break; + } + start_after = t.rbegin()->first; + m->insert(t.begin(), t.end()); + } while (more); + return 0; +} + +int RGWSI_SysObj_Core::omap_set(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, const std::string& key, + bufferlist& bl, bool must_exist, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + ldpp_dout(dpp, 15) << "omap_set obj=" << obj << " key=" << key << dendl; + + map<string, bufferlist> m; + m[key] = bl; + librados::ObjectWriteOperation op; + if (must_exist) + op.assert_exists(); + op.omap_set(m); + r = rados_obj.operate(dpp, &op, y); + return r; +} + +int RGWSI_SysObj_Core::omap_set(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + const std::map<std::string, bufferlist>& m, + bool must_exist, optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + librados::ObjectWriteOperation op; + if (must_exist) + op.assert_exists(); + op.omap_set(m); + r = rados_obj.operate(dpp, &op, y); + return r; +} + +int RGWSI_SysObj_Core::omap_del(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, const std::string& key, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + set<string> k; + k.insert(key); + + librados::ObjectWriteOperation op; + + op.omap_rm_keys(k); + + r = rados_obj.operate(dpp, &op, y); + return r; +} + +int RGWSI_SysObj_Core::notify(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, bufferlist& bl, + uint64_t timeout_ms, bufferlist *pbl, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + r = rados_obj.notify(dpp, bl, timeout_ms, pbl, y); + return r; +} + +int RGWSI_SysObj_Core::remove(const DoutPrefixProvider *dpp, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + librados::ObjectWriteOperation op; + + if (objv_tracker) { + objv_tracker->prepare_op_for_write(&op); + } + + op.remove(); + r = rados_obj.operate(dpp, &op, y); + if (r < 0) + return r; + + return 0; +} + +int RGWSI_SysObj_Core::write(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + real_time *pmtime, + map<std::string, bufferlist>& attrs, + bool exclusive, + const bufferlist& data, + RGWObjVersionTracker *objv_tracker, + real_time set_mtime, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + librados::ObjectWriteOperation op; + + if (exclusive) { + op.create(true); // exclusive create + } else { + op.remove(); + op.set_op_flags2(LIBRADOS_OP_FLAG_FAILOK); + op.create(false); + } + + if (objv_tracker) { + objv_tracker->prepare_op_for_write(&op); + } + + if (real_clock::is_zero(set_mtime)) { + set_mtime = real_clock::now(); + } + + struct timespec mtime_ts = real_clock::to_timespec(set_mtime); + op.mtime2(&mtime_ts); + op.write_full(data); + + bufferlist acl_bl; + + for (map<string, bufferlist>::iterator iter = attrs.begin(); iter != attrs.end(); ++iter) { + const string& name = iter->first; + bufferlist& bl = iter->second; + + if (!bl.length()) + continue; + + op.setxattr(name.c_str(), bl); + } + + r = rados_obj.operate(dpp, &op, y); + if (r < 0) { + return r; + } + + if (objv_tracker) { + objv_tracker->apply_write(); + } + + if (pmtime) { + *pmtime = set_mtime; + } + + return 0; +} + + +int RGWSI_SysObj_Core::write_data(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const bufferlist& bl, + bool exclusive, + RGWObjVersionTracker *objv_tracker, + optional_yield y) +{ + RGWSI_RADOS::Obj rados_obj; + int r = get_rados_obj(dpp, zone_svc, obj, &rados_obj); + if (r < 0) { + ldpp_dout(dpp, 20) << "get_rados_obj() on obj=" << obj << " returned " << r << dendl; + return r; + } + + librados::ObjectWriteOperation op; + + if (exclusive) { + op.create(true); + } + + if (objv_tracker) { + objv_tracker->prepare_op_for_write(&op); + } + op.write_full(bl); + r = rados_obj.operate(dpp, &op, y); + if (r < 0) + return r; + + if (objv_tracker) { + objv_tracker->apply_write(); + } + return 0; +} + +int RGWSI_SysObj_Core::pool_list_prefixed_objs(const DoutPrefixProvider *dpp, + const rgw_pool& pool, const string& prefix, + std::function<void(const string&)> cb) +{ + bool is_truncated; + + auto rados_pool = rados_svc->pool(pool); + + auto op = rados_pool.op(); + + RGWAccessListFilterPrefix filter(prefix); + + int r = op.init(dpp, string(), &filter); + if (r < 0) { + return r; + } + + do { + vector<string> oids; +#define MAX_OBJS_DEFAULT 1000 + int r = op.get_next(dpp, MAX_OBJS_DEFAULT, &oids, &is_truncated); + if (r < 0) { + return r; + } + for (auto& val : oids) { + if (val.size() > prefix.size()) { + cb(val.substr(prefix.size())); + } + } + } while (is_truncated); + + return 0; +} + +int RGWSI_SysObj_Core::pool_list_objects_init(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + const string& marker, + const string& prefix, + RGWSI_SysObj::Pool::ListCtx *_ctx) +{ + _ctx->impl.emplace<PoolListImplInfo>(prefix); + + auto& ctx = static_cast<PoolListImplInfo&>(*_ctx->impl); + + ctx.pool = rados_svc->pool(pool); + ctx.op = ctx.pool.op(); + + int r = ctx.op.init(dpp, marker, &ctx.filter); + if (r < 0) { + ldpp_dout(dpp, 10) << "failed to list objects pool_iterate_begin() returned r=" << r << dendl; + return r; + } + return 0; +} + +int RGWSI_SysObj_Core::pool_list_objects_next(const DoutPrefixProvider *dpp, + RGWSI_SysObj::Pool::ListCtx& _ctx, + int max, + vector<string> *oids, + bool *is_truncated) +{ + if (!_ctx.impl) { + return -EINVAL; + } + auto& ctx = static_cast<PoolListImplInfo&>(*_ctx.impl); + int r = ctx.op.get_next(dpp, max, oids, is_truncated); + if (r < 0) { + if(r != -ENOENT) + ldpp_dout(dpp, 10) << "failed to list objects pool_iterate returned r=" << r << dendl; + return r; + } + + return oids->size(); +} + +int RGWSI_SysObj_Core::pool_list_objects_get_marker(RGWSI_SysObj::Pool::ListCtx& _ctx, + string *marker) +{ + if (!_ctx.impl) { + return -EINVAL; + } + + auto& ctx = static_cast<PoolListImplInfo&>(*_ctx.impl); + return ctx.op.get_marker(marker); +} diff --git a/src/rgw/services/svc_sys_obj_core.h b/src/rgw/services/svc_sys_obj_core.h new file mode 100644 index 000000000..d02a37eee --- /dev/null +++ b/src/rgw/services/svc_sys_obj_core.h @@ -0,0 +1,145 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" + +#include "svc_rados.h" +#include "svc_sys_obj.h" +#include "svc_sys_obj_core_types.h" + + +class RGWSI_Zone; + +struct rgw_cache_entry_info; + +class RGWSI_SysObj_Core : public RGWServiceInstance +{ + friend class RGWServices_Def; + friend class RGWSI_SysObj; + +protected: + RGWSI_RADOS *rados_svc{nullptr}; + RGWSI_Zone *zone_svc{nullptr}; + + using GetObjState = RGWSI_SysObj_Core_GetObjState; + using PoolListImplInfo = RGWSI_SysObj_Core_PoolListImplInfo; + + void core_init(RGWSI_RADOS *_rados_svc, + RGWSI_Zone *_zone_svc) { + rados_svc = _rados_svc; + zone_svc = _zone_svc; + } + int get_rados_obj(const DoutPrefixProvider *dpp, RGWSI_Zone *zone_svc, const rgw_raw_obj& obj, RGWSI_RADOS::Obj *pobj); + + virtual int raw_stat(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + uint64_t *psize, real_time *pmtime, + std::map<std::string, bufferlist> *attrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y); + + virtual int read(const DoutPrefixProvider *dpp, + RGWSI_SysObj_Obj_GetObjState& read_state, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + bufferlist *bl, off_t ofs, off_t end, + ceph::real_time* pmtime, uint64_t* psize, + std::map<std::string, bufferlist> *attrs, + bool raw_attrs, + rgw_cache_entry_info *cache_info, + boost::optional<obj_version>, + optional_yield y); + + virtual int remove(const DoutPrefixProvider *dpp, + RGWObjVersionTracker *objv_tracker, + const rgw_raw_obj& obj, + optional_yield y); + + virtual int write(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + real_time *pmtime, + std::map<std::string, bufferlist>& attrs, + bool exclusive, + const bufferlist& data, + RGWObjVersionTracker *objv_tracker, + real_time set_mtime, + optional_yield y); + + virtual int write_data(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const bufferlist& bl, + bool exclusive, + RGWObjVersionTracker *objv_tracker, + optional_yield y); + + virtual int get_attr(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + const char *name, bufferlist *dest, + optional_yield y); + + virtual int set_attrs(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + std::map<std::string, bufferlist>& attrs, + std::map<std::string, bufferlist> *rmattrs, + RGWObjVersionTracker *objv_tracker, + bool exclusive, optional_yield y); + + virtual int omap_get_all(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, std::map<std::string, bufferlist> *m, + optional_yield y); + virtual int omap_get_vals(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, + const std::string& marker, + uint64_t count, + std::map<std::string, bufferlist> *m, + bool *pmore, + optional_yield y); + virtual int omap_set(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, const std::string& key, + bufferlist& bl, bool must_exist, + optional_yield y); + virtual int omap_set(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, + const std::map<std::string, bufferlist>& m, bool must_exist, + optional_yield y); + virtual int omap_del(const DoutPrefixProvider *dpp, const rgw_raw_obj& obj, const std::string& key, + optional_yield y); + + virtual int notify(const DoutPrefixProvider *dpp, + const rgw_raw_obj& obj, bufferlist& bl, + uint64_t timeout_ms, bufferlist *pbl, + optional_yield y); + + virtual int pool_list_prefixed_objs(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + const std::string& prefix, + std::function<void(const std::string&)> cb); + + virtual int pool_list_objects_init(const DoutPrefixProvider *dpp, + const rgw_pool& pool, + const std::string& marker, + const std::string& prefix, + RGWSI_SysObj::Pool::ListCtx *ctx); + virtual int pool_list_objects_next(const DoutPrefixProvider *dpp, + RGWSI_SysObj::Pool::ListCtx& ctx, + int max, + std::vector<std::string> *oids, + bool *is_truncated); + + virtual int pool_list_objects_get_marker(RGWSI_SysObj::Pool::ListCtx& _ctx, + std::string *marker); + + int stat(RGWSI_SysObj_Obj_GetObjState& state, + const rgw_raw_obj& obj, + std::map<std::string, bufferlist> *attrs, + bool raw_attrs, + real_time *lastmod, + uint64_t *obj_size, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp); + +public: + RGWSI_SysObj_Core(CephContext *cct): RGWServiceInstance(cct) {} + + RGWSI_Zone *get_zone_svc() { + return zone_svc; + } +}; diff --git a/src/rgw/services/svc_sys_obj_core_types.h b/src/rgw/services/svc_sys_obj_core_types.h new file mode 100644 index 000000000..74f489d91 --- /dev/null +++ b/src/rgw/services/svc_sys_obj_core_types.h @@ -0,0 +1,34 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + + +#include "rgw_service.h" + +#include "svc_rados.h" +#include "svc_sys_obj_types.h" + + + +struct RGWSI_SysObj_Core_GetObjState : public RGWSI_SysObj_Obj_GetObjState { + RGWSI_RADOS::Obj rados_obj; + bool has_rados_obj{false}; + uint64_t last_ver{0}; + + RGWSI_SysObj_Core_GetObjState() {} + + int get_rados_obj(const DoutPrefixProvider *dpp, + RGWSI_RADOS *rados_svc, + RGWSI_Zone *zone_svc, + const rgw_raw_obj& obj, + RGWSI_RADOS::Obj **pobj); +}; + +struct RGWSI_SysObj_Core_PoolListImplInfo : public RGWSI_SysObj_Pool_ListInfo { + RGWSI_RADOS::Pool pool; + RGWSI_RADOS::Pool::List op; + RGWAccessListFilterPrefix filter; + + RGWSI_SysObj_Core_PoolListImplInfo(const std::string& prefix) : op(pool.op()), filter(prefix) {} +}; diff --git a/src/rgw/services/svc_sys_obj_types.h b/src/rgw/services/svc_sys_obj_types.h new file mode 100644 index 000000000..b5bc2d40d --- /dev/null +++ b/src/rgw/services/svc_sys_obj_types.h @@ -0,0 +1,15 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + + +#pragma once + + +#include "rgw_service.h" + + +struct RGWSI_SysObj_Obj_GetObjState { +}; + +struct RGWSI_SysObj_Pool_ListInfo { +}; diff --git a/src/rgw/services/svc_tier_rados.cc b/src/rgw/services/svc_tier_rados.cc new file mode 100644 index 000000000..ca87e8ace --- /dev/null +++ b/src/rgw/services/svc_tier_rados.cc @@ -0,0 +1,36 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_tier_rados.h" + +using namespace std; + +const std::string MP_META_SUFFIX = ".meta"; + +MultipartMetaFilter::~MultipartMetaFilter() {} + +bool MultipartMetaFilter::filter(const string& name, string& key) { + // the length of the suffix so we can skip past it + static const size_t MP_META_SUFFIX_LEN = MP_META_SUFFIX.length(); + + size_t len = name.size(); + + // make sure there's room for suffix plus at least one more + // character + if (len <= MP_META_SUFFIX_LEN) + return false; + + size_t pos = name.find(MP_META_SUFFIX, len - MP_META_SUFFIX_LEN); + if (pos == string::npos) + return false; + + pos = name.rfind('.', pos - 1); + if (pos == string::npos) + return false; + + key = name.substr(0, pos); + + return true; +} + + diff --git a/src/rgw/services/svc_tier_rados.h b/src/rgw/services/svc_tier_rados.h new file mode 100644 index 000000000..a2036b933 --- /dev/null +++ b/src/rgw/services/svc_tier_rados.h @@ -0,0 +1,154 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include <iomanip> + +#include "rgw_service.h" + +#include "svc_rados.h" + +extern const std::string MP_META_SUFFIX; + +class RGWMPObj { + std::string oid; + std::string prefix; + std::string meta; + std::string upload_id; +public: + RGWMPObj() {} + RGWMPObj(const std::string& _oid, const std::string& _upload_id) { + init(_oid, _upload_id, _upload_id); + } + RGWMPObj(const std::string& _oid, std::optional<std::string> _upload_id) { + if (_upload_id) { + init(_oid, *_upload_id, *_upload_id); + } else { + from_meta(_oid); + } + } + void init(const std::string& _oid, const std::string& _upload_id) { + init(_oid, _upload_id, _upload_id); + } + void init(const std::string& _oid, const std::string& _upload_id, const std::string& part_unique_str) { + if (_oid.empty()) { + clear(); + return; + } + oid = _oid; + upload_id = _upload_id; + prefix = oid + "."; + meta = prefix + upload_id + MP_META_SUFFIX; + prefix.append(part_unique_str); + } + const std::string& get_meta() const { return meta; } + std::string get_part(int num) const { + char buf[16]; + snprintf(buf, 16, ".%d", num); + std::string s = prefix; + s.append(buf); + return s; + } + std::string get_part(const std::string& part) const { + std::string s = prefix; + s.append("."); + s.append(part); + return s; + } + const std::string& get_upload_id() const { + return upload_id; + } + const std::string& get_key() const { + return oid; + } + bool from_meta(const std::string& meta) { + int end_pos = meta.rfind('.'); // search for ".meta" + if (end_pos < 0) + return false; + int mid_pos = meta.rfind('.', end_pos - 1); // <key>.<upload_id> + if (mid_pos < 0) + return false; + oid = meta.substr(0, mid_pos); + upload_id = meta.substr(mid_pos + 1, end_pos - mid_pos - 1); + init(oid, upload_id, upload_id); + return true; + } + void clear() { + oid = ""; + prefix = ""; + meta = ""; + upload_id = ""; + } + friend std::ostream& operator<<(std::ostream& out, const RGWMPObj& obj) { + return out << "RGWMPObj:{ prefix=" << std::quoted(obj.prefix) << + ", meta=" << std::quoted(obj.meta) << " }"; + } +}; // class RGWMPObj + +/** + * A filter to a) test whether an object name is a multipart meta + * object, and b) filter out just the key used to determine the bucket + * index shard. + * + * Objects for multipart meta have names adorned with an upload id and + * other elements -- specifically a ".", MULTIPART_UPLOAD_ID_PREFIX, + * unique id, and MP_META_SUFFIX. This filter will return true when + * the name provided is such. It will also extract the key used for + * bucket index shard calculation from the adorned name. + */ +class MultipartMetaFilter : public RGWAccessListFilter { +public: + MultipartMetaFilter() {} + + virtual ~MultipartMetaFilter() override; + + /** + * @param name [in] The object name as it appears in the bucket index. + * @param key [out] An output parameter that will contain the bucket + * index key if this entry is in the form of a multipart meta object. + * @return true if the name provided is in the form of a multipart meta + * object, false otherwise + */ + bool filter(const std::string& name, std::string& key) override; +}; + +class RGWSI_Tier_RADOS : public RGWServiceInstance +{ + RGWSI_Zone *zone_svc{nullptr}; + +public: + RGWSI_Tier_RADOS(CephContext *cct): RGWServiceInstance(cct) {} + + void init(RGWSI_Zone *_zone_svc) { + zone_svc = _zone_svc; + } + + static inline bool raw_obj_to_obj(const rgw_bucket& bucket, const rgw_raw_obj& raw_obj, rgw_obj *obj) { + ssize_t pos = raw_obj.oid.find('_', bucket.marker.length()); + if (pos < 0) { + return false; + } + + if (!rgw_obj_key::parse_raw_oid(raw_obj.oid.substr(pos + 1), &obj->key)) { + return false; + } + obj->bucket = bucket; + + return true; + } +}; + diff --git a/src/rgw/services/svc_user.cc b/src/rgw/services/svc_user.cc new file mode 100644 index 000000000..9a07c207b --- /dev/null +++ b/src/rgw/services/svc_user.cc @@ -0,0 +1,11 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + + +#include "svc_user.h" + +RGWSI_User::RGWSI_User(CephContext *cct): RGWServiceInstance(cct) { +} + +RGWSI_User::~RGWSI_User() { +} diff --git a/src/rgw/services/svc_user.h b/src/rgw/services/svc_user.h new file mode 100644 index 000000000..1cb459d31 --- /dev/null +++ b/src/rgw/services/svc_user.h @@ -0,0 +1,127 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "svc_meta_be.h" + +#include "rgw_service.h" + +class RGWUserBuckets; +class RGWGetUserStats_CB; + +class RGWSI_User : public RGWServiceInstance +{ +public: + RGWSI_User(CephContext *cct); + virtual ~RGWSI_User(); + + static std::string get_meta_key(const rgw_user& user) { + return user.to_str(); + } + + static rgw_user user_from_meta_key(const std::string& key) { + return rgw_user(key); + } + + virtual RGWSI_MetaBackend_Handler *get_be_handler() = 0; + + /* base svc_user interfaces */ + + virtual int read_user_info(RGWSI_MetaBackend::Context *ctx, + const rgw_user& user, + RGWUserInfo *info, + RGWObjVersionTracker * const objv_tracker, + real_time * const pmtime, + rgw_cache_entry_info * const cache_info, + std::map<std::string, bufferlist> * const pattrs, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int store_user_info(RGWSI_MetaBackend::Context *ctx, + const RGWUserInfo& info, + RGWUserInfo *old_info, + RGWObjVersionTracker *objv_tracker, + const real_time& mtime, + bool exclusive, + std::map<std::string, bufferlist> *attrs, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int remove_user_info(RGWSI_MetaBackend::Context *ctx, + const RGWUserInfo& info, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int get_user_info_by_email(RGWSI_MetaBackend::Context *ctx, + const std::string& email, RGWUserInfo *info, + RGWObjVersionTracker *objv_tracker, + real_time *pmtime, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + virtual int get_user_info_by_swift(RGWSI_MetaBackend::Context *ctx, + const std::string& swift_name, + RGWUserInfo *info, /* out */ + RGWObjVersionTracker * const objv_tracker, + real_time * const pmtime, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + virtual int get_user_info_by_access_key(RGWSI_MetaBackend::Context *ctx, + const std::string& access_key, + RGWUserInfo *info, + RGWObjVersionTracker* objv_tracker, + real_time *pmtime, + optional_yield y, + const DoutPrefixProvider *dpp) = 0; + + virtual int add_bucket(const DoutPrefixProvider *dpp, + const rgw_user& user, + const rgw_bucket& bucket, + ceph::real_time creation_time, + optional_yield y) = 0; + virtual int remove_bucket(const DoutPrefixProvider *dpp, + const rgw_user& user, + const rgw_bucket& _bucket, optional_yield) = 0; + virtual int list_buckets(const DoutPrefixProvider *dpp, + const rgw_user& user, + const std::string& marker, + const std::string& end_marker, + uint64_t max, + RGWUserBuckets *buckets, + bool *is_truncated, + optional_yield y) = 0; + + virtual int flush_bucket_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, + const RGWBucketEnt& ent, optional_yield y) = 0; + virtual int complete_flush_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, optional_yield y) = 0; + virtual int reset_bucket_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, + optional_yield y) = 0; + virtual int read_stats(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const rgw_user& user, RGWStorageStats *stats, + ceph::real_time *last_stats_sync, /* last time a full stats sync completed */ + ceph::real_time *last_stats_update, + optional_yield y) = 0; /* last time a stats update was done */ + + virtual int read_stats_async(const DoutPrefixProvider *dpp, + const rgw_user& user, RGWGetUserStats_CB *cb) = 0; +}; + diff --git a/src/rgw/services/svc_user_rados.cc b/src/rgw/services/svc_user_rados.cc new file mode 100644 index 000000000..c99af9354 --- /dev/null +++ b/src/rgw/services/svc_user_rados.cc @@ -0,0 +1,968 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include <boost/algorithm/string.hpp> + +#include "svc_user.h" +#include "svc_user_rados.h" +#include "svc_zone.h" +#include "svc_sys_obj.h" +#include "svc_sys_obj_cache.h" +#include "svc_meta.h" +#include "svc_meta_be_sobj.h" +#include "svc_sync_modules.h" + +#include "rgw_user.h" +#include "rgw_bucket.h" +#include "rgw_tools.h" +#include "rgw_zone.h" +#include "rgw_rados.h" + +#include "cls/user/cls_user_client.h" + +#define dout_subsys ceph_subsys_rgw + +#define RGW_BUCKETS_OBJ_SUFFIX ".buckets" + +using namespace std; + +class RGWSI_User_Module : public RGWSI_MBSObj_Handler_Module { + RGWSI_User_RADOS::Svc& svc; + + const string prefix; +public: + RGWSI_User_Module(RGWSI_User_RADOS::Svc& _svc) : RGWSI_MBSObj_Handler_Module("user"), + svc(_svc) {} + + void get_pool_and_oid(const string& key, rgw_pool *pool, string *oid) override { + if (pool) { + *pool = svc.zone->get_zone_params().user_uid_pool; + } + if (oid) { + *oid = key; + } + } + + const string& get_oid_prefix() override { + return prefix; + } + + bool is_valid_oid(const string& oid) override { + // filter out the user.buckets objects + return !boost::algorithm::ends_with(oid, RGW_BUCKETS_OBJ_SUFFIX); + } + + string key_to_oid(const string& key) override { + return key; + } + + string oid_to_key(const string& oid) override { + return oid; + } +}; + +RGWSI_User_RADOS::RGWSI_User_RADOS(CephContext *cct): RGWSI_User(cct) { +} + +RGWSI_User_RADOS::~RGWSI_User_RADOS() { +} + +void RGWSI_User_RADOS::init(RGWSI_RADOS *_rados_svc, + RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, + RGWSI_SysObj_Cache *_cache_svc, RGWSI_Meta *_meta_svc, + RGWSI_MetaBackend *_meta_be_svc, + RGWSI_SyncModules *_sync_modules_svc) +{ + svc.user = this; + svc.rados = _rados_svc; + svc.zone = _zone_svc; + svc.sysobj = _sysobj_svc; + svc.cache = _cache_svc; + svc.meta = _meta_svc; + svc.meta_be = _meta_be_svc; + svc.sync_modules = _sync_modules_svc; +} + +int RGWSI_User_RADOS::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + uinfo_cache.reset(new RGWChainedCacheImpl<user_info_cache_entry>); + uinfo_cache->init(svc.cache); + + int r = svc.meta->create_be_handler(RGWSI_MetaBackend::Type::MDBE_SOBJ, &be_handler); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to create be handler: r=" << r << dendl; + return r; + } + + RGWSI_MetaBackend_Handler_SObj *bh = static_cast<RGWSI_MetaBackend_Handler_SObj *>(be_handler); + + auto module = new RGWSI_User_Module(svc); + be_module.reset(module); + bh->set_module(module); + return 0; +} + +rgw_raw_obj RGWSI_User_RADOS::get_buckets_obj(const rgw_user& user) const +{ + string oid = user.to_str() + RGW_BUCKETS_OBJ_SUFFIX; + return rgw_raw_obj(svc.zone->get_zone_params().user_uid_pool, oid); +} + +int RGWSI_User_RADOS::read_user_info(RGWSI_MetaBackend::Context *ctx, + const rgw_user& user, + RGWUserInfo *info, + RGWObjVersionTracker * const objv_tracker, + real_time * const pmtime, + rgw_cache_entry_info * const cache_info, + map<string, bufferlist> * const pattrs, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + if(user.id == RGW_USER_ANON_ID) { + ldpp_dout(dpp, 20) << "RGWSI_User_RADOS::read_user_info(): anonymous user" << dendl; + return -ENOENT; + } + bufferlist bl; + RGWUID user_id; + + RGWSI_MBSObj_GetParams params(&bl, pattrs, pmtime); + params.set_cache_info(cache_info); + + int ret = svc.meta_be->get_entry(ctx, get_meta_key(user), params, objv_tracker, y, dpp); + if (ret < 0) { + return ret; + } + + auto iter = bl.cbegin(); + try { + decode(user_id, iter); + if (user_id.user_id != user) { + ldpp_dout(dpp, -1) << "ERROR: rgw_get_user_info_by_uid(): user id mismatch: " << user_id.user_id << " != " << user << dendl; + return -EIO; + } + if (!iter.end()) { + decode(*info, iter); + } + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: failed to decode user info, caught buffer::error" << dendl; + return -EIO; + } + + return 0; +} + +class PutOperation +{ + RGWSI_User_RADOS::Svc& svc; + RGWSI_MetaBackend_SObj::Context_SObj *ctx; + RGWUID ui; + const RGWUserInfo& info; + RGWUserInfo *old_info; + RGWObjVersionTracker *objv_tracker; + const real_time& mtime; + bool exclusive; + map<string, bufferlist> *pattrs; + RGWObjVersionTracker ot; + string err_msg; + optional_yield y; + + void set_err_msg(string msg) { + if (!err_msg.empty()) { + err_msg = std::move(msg); + } + } + +public: + PutOperation(RGWSI_User_RADOS::Svc& svc, + RGWSI_MetaBackend::Context *_ctx, + const RGWUserInfo& info, + RGWUserInfo *old_info, + RGWObjVersionTracker *objv_tracker, + const real_time& mtime, + bool exclusive, + map<string, bufferlist> *pattrs, + optional_yield y) : + svc(svc), info(info), old_info(old_info), + objv_tracker(objv_tracker), mtime(mtime), + exclusive(exclusive), pattrs(pattrs), y(y) { + ctx = static_cast<RGWSI_MetaBackend_SObj::Context_SObj *>(_ctx); + ui.user_id = info.user_id; + } + + int prepare(const DoutPrefixProvider *dpp) { + if (objv_tracker) { + ot = *objv_tracker; + } + + if (ot.write_version.tag.empty()) { + if (ot.read_version.tag.empty()) { + ot.generate_new_write_ver(svc.meta_be->ctx()); + } else { + ot.write_version = ot.read_version; + ot.write_version.ver++; + } + } + + for (auto iter = info.swift_keys.begin(); iter != info.swift_keys.end(); ++iter) { + if (old_info && old_info->swift_keys.count(iter->first) != 0) + continue; + auto& k = iter->second; + /* check if swift mapping exists */ + RGWUserInfo inf; + int r = svc.user->get_user_info_by_swift(ctx, k.id, &inf, nullptr, nullptr, y, dpp); + if (r >= 0 && inf.user_id != info.user_id && + (!old_info || inf.user_id != old_info->user_id)) { + ldpp_dout(dpp, 0) << "WARNING: can't store user info, swift id (" << k.id + << ") already mapped to another user (" << info.user_id << ")" << dendl; + return -EEXIST; + } + } + + /* check if access keys already exist */ + for (auto iter = info.access_keys.begin(); iter != info.access_keys.end(); ++iter) { + if (old_info && old_info->access_keys.count(iter->first) != 0) + continue; + auto& k = iter->second; + RGWUserInfo inf; + int r = svc.user->get_user_info_by_access_key(ctx, k.id, &inf, nullptr, nullptr, y, dpp); + if (r >= 0 && inf.user_id != info.user_id && + (!old_info || inf.user_id != old_info->user_id)) { + ldpp_dout(dpp, 0) << "WARNING: can't store user info, access key already mapped to another user" << dendl; + return -EEXIST; + } + } + + return 0; + } + + int put(const DoutPrefixProvider *dpp) { + bufferlist data_bl; + encode(ui, data_bl); + encode(info, data_bl); + + RGWSI_MBSObj_PutParams params(data_bl, pattrs, mtime, exclusive); + + int ret = svc.meta_be->put(ctx, RGWSI_User::get_meta_key(info.user_id), params, &ot, y, dpp); + if (ret < 0) + return ret; + + return 0; + } + + int complete(const DoutPrefixProvider *dpp) { + int ret; + + bufferlist link_bl; + encode(ui, link_bl); + + if (!info.user_email.empty()) { + if (!old_info || + old_info->user_email.compare(info.user_email) != 0) { /* only if new index changed */ + ret = rgw_put_system_obj(dpp, svc.sysobj, svc.zone->get_zone_params().user_email_pool, info.user_email, + link_bl, exclusive, NULL, real_time(), y); + if (ret < 0) + return ret; + } + } + + const bool renamed = old_info && old_info->user_id != info.user_id; + for (auto iter = info.access_keys.begin(); iter != info.access_keys.end(); ++iter) { + auto& k = iter->second; + if (old_info && old_info->access_keys.count(iter->first) != 0 && !renamed) + continue; + + ret = rgw_put_system_obj(dpp, svc.sysobj, svc.zone->get_zone_params().user_keys_pool, k.id, + link_bl, exclusive, NULL, real_time(), y); + if (ret < 0) + return ret; + } + + for (auto siter = info.swift_keys.begin(); siter != info.swift_keys.end(); ++siter) { + auto& k = siter->second; + if (old_info && old_info->swift_keys.count(siter->first) != 0 && !renamed) + continue; + + ret = rgw_put_system_obj(dpp, svc.sysobj, svc.zone->get_zone_params().user_swift_pool, k.id, + link_bl, exclusive, NULL, real_time(), y); + if (ret < 0) + return ret; + } + + if (old_info) { + ret = remove_old_indexes(*old_info, info, y, dpp); + if (ret < 0) { + return ret; + } + } + + return 0; + } + + int remove_old_indexes(const RGWUserInfo& old_info, const RGWUserInfo& new_info, optional_yield y, const DoutPrefixProvider *dpp) { + int ret; + + if (!old_info.user_id.empty() && + old_info.user_id != new_info.user_id) { + if (old_info.user_id.tenant != new_info.user_id.tenant) { + ldpp_dout(dpp, 0) << "ERROR: tenant mismatch: " << old_info.user_id.tenant << " != " << new_info.user_id.tenant << dendl; + return -EINVAL; + } + ret = svc.user->remove_uid_index(ctx, old_info, nullptr, y, dpp); + if (ret < 0 && ret != -ENOENT) { + set_err_msg("ERROR: could not remove index for uid " + old_info.user_id.to_str()); + return ret; + } + } + + if (!old_info.user_email.empty() && + old_info.user_email != new_info.user_email) { + ret = svc.user->remove_email_index(dpp, old_info.user_email, y); + if (ret < 0 && ret != -ENOENT) { + set_err_msg("ERROR: could not remove index for email " + old_info.user_email); + return ret; + } + } + + for ([[maybe_unused]] const auto& [name, access_key] : old_info.access_keys) { + if (!new_info.access_keys.count(access_key.id)) { + ret = svc.user->remove_key_index(dpp, access_key, y); + if (ret < 0 && ret != -ENOENT) { + set_err_msg("ERROR: could not remove index for key " + access_key.id); + return ret; + } + } + } + + for (auto old_iter = old_info.swift_keys.begin(); old_iter != old_info.swift_keys.end(); ++old_iter) { + const auto& swift_key = old_iter->second; + auto new_iter = new_info.swift_keys.find(swift_key.id); + if (new_iter == new_info.swift_keys.end()) { + ret = svc.user->remove_swift_name_index(dpp, swift_key.id, y); + if (ret < 0 && ret != -ENOENT) { + set_err_msg("ERROR: could not remove index for swift_name " + swift_key.id); + return ret; + } + } + } + + return 0; + } + + const string& get_err_msg() { + return err_msg; + } +}; + +int RGWSI_User_RADOS::store_user_info(RGWSI_MetaBackend::Context *ctx, + const RGWUserInfo& info, + RGWUserInfo *old_info, + RGWObjVersionTracker *objv_tracker, + const real_time& mtime, + bool exclusive, + map<string, bufferlist> *attrs, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + PutOperation op(svc, ctx, + info, old_info, + objv_tracker, + mtime, exclusive, + attrs, + y); + + int r = op.prepare(dpp); + if (r < 0) { + return r; + } + + r = op.put(dpp); + if (r < 0) { + return r; + } + + r = op.complete(dpp); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_User_RADOS::remove_key_index(const DoutPrefixProvider *dpp, + const RGWAccessKey& access_key, + optional_yield y) +{ + rgw_raw_obj obj(svc.zone->get_zone_params().user_keys_pool, access_key.id); + auto sysobj = svc.sysobj->get_obj(obj); + return sysobj.wop().remove(dpp, y); +} + +int RGWSI_User_RADOS::remove_email_index(const DoutPrefixProvider *dpp, + const string& email, + optional_yield y) +{ + if (email.empty()) { + return 0; + } + rgw_raw_obj obj(svc.zone->get_zone_params().user_email_pool, email); + auto sysobj = svc.sysobj->get_obj(obj); + return sysobj.wop().remove(dpp, y); +} + +int RGWSI_User_RADOS::remove_swift_name_index(const DoutPrefixProvider *dpp, + const string& swift_name, + optional_yield y) +{ + rgw_raw_obj obj(svc.zone->get_zone_params().user_swift_pool, swift_name); + auto sysobj = svc.sysobj->get_obj(obj); + return sysobj.wop().remove(dpp, y); +} + +/** + * delete a user's presence from the RGW system. + * First remove their bucket ACLs, then delete them + * from the user and user email pools. This leaves the pools + * themselves alone, as well as any ACLs embedded in object xattrs. + */ +int RGWSI_User_RADOS::remove_user_info(RGWSI_MetaBackend::Context *ctx, + const RGWUserInfo& info, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + int ret; + + auto kiter = info.access_keys.begin(); + for (; kiter != info.access_keys.end(); ++kiter) { + ldpp_dout(dpp, 10) << "removing key index: " << kiter->first << dendl; + ret = remove_key_index(dpp, kiter->second, y); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: could not remove " << kiter->first << " (access key object), should be fixed (err=" << ret << ")" << dendl; + return ret; + } + } + + auto siter = info.swift_keys.begin(); + for (; siter != info.swift_keys.end(); ++siter) { + auto& k = siter->second; + ldpp_dout(dpp, 10) << "removing swift subuser index: " << k.id << dendl; + /* check if swift mapping exists */ + ret = remove_swift_name_index(dpp, k.id, y); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: could not remove " << k.id << " (swift name object), should be fixed (err=" << ret << ")" << dendl; + return ret; + } + } + + ldpp_dout(dpp, 10) << "removing email index: " << info.user_email << dendl; + ret = remove_email_index(dpp, info.user_email, y); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: could not remove email index object for " + << info.user_email << ", should be fixed (err=" << ret << ")" << dendl; + return ret; + } + + rgw_raw_obj uid_bucks = get_buckets_obj(info.user_id); + ldpp_dout(dpp, 10) << "removing user buckets index" << dendl; + auto sysobj = svc.sysobj->get_obj(uid_bucks); + ret = sysobj.wop().remove(dpp, y); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: could not remove " << info.user_id << ":" << uid_bucks << ", should be fixed (err=" << ret << ")" << dendl; + return ret; + } + + ret = remove_uid_index(ctx, info, objv_tracker, y, dpp); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + + return 0; +} + +int RGWSI_User_RADOS::remove_uid_index(RGWSI_MetaBackend::Context *ctx, const RGWUserInfo& user_info, RGWObjVersionTracker *objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) +{ + ldpp_dout(dpp, 10) << "removing user index: " << user_info.user_id << dendl; + + RGWSI_MBSObj_RemoveParams params; + int ret = svc.meta_be->remove(ctx, get_meta_key(user_info.user_id), params, objv_tracker, y, dpp); + if (ret < 0 && ret != -ENOENT && ret != -ECANCELED) { + string key; + user_info.user_id.to_str(key); + rgw_raw_obj uid_obj(svc.zone->get_zone_params().user_uid_pool, key); + ldpp_dout(dpp, 0) << "ERROR: could not remove " << user_info.user_id << ":" << uid_obj << ", should be fixed (err=" << ret << ")" << dendl; + return ret; + } + + return 0; +} + +int RGWSI_User_RADOS::get_user_info_from_index(RGWSI_MetaBackend::Context* ctx, + const string& key, + const rgw_pool& pool, + RGWUserInfo *info, + RGWObjVersionTracker* objv_tracker, + real_time* pmtime, optional_yield y, + const DoutPrefixProvider* dpp) +{ + string cache_key = pool.to_str() + "/" + key; + + if (auto e = uinfo_cache->find(cache_key)) { + *info = e->info; + if (objv_tracker) + *objv_tracker = e->objv_tracker; + if (pmtime) + *pmtime = e->mtime; + return 0; + } + + user_info_cache_entry e; + bufferlist bl; + RGWUID uid; + + int ret = rgw_get_system_obj(svc.sysobj, pool, key, bl, nullptr, &e.mtime, y, dpp); + if (ret < 0) + return ret; + + rgw_cache_entry_info cache_info; + + auto iter = bl.cbegin(); + try { + decode(uid, iter); + + int ret = read_user_info(ctx, uid.user_id, + &e.info, &e.objv_tracker, nullptr, &cache_info, nullptr, + y, dpp); + if (ret < 0) { + return ret; + } + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: failed to decode user info, caught buffer::error" << dendl; + return -EIO; + } + + uinfo_cache->put(dpp, svc.cache, cache_key, &e, { &cache_info }); + + *info = e.info; + if (objv_tracker) + *objv_tracker = e.objv_tracker; + if (pmtime) + *pmtime = e.mtime; + + return 0; +} + +/** + * Given an email, finds the user info associated with it. + * returns: 0 on success, -ERR# on failure (including nonexistence) + */ +int RGWSI_User_RADOS::get_user_info_by_email(RGWSI_MetaBackend::Context *ctx, + const string& email, RGWUserInfo *info, + RGWObjVersionTracker *objv_tracker, + real_time *pmtime, optional_yield y, + const DoutPrefixProvider *dpp) +{ + return get_user_info_from_index(ctx, email, svc.zone->get_zone_params().user_email_pool, + info, objv_tracker, pmtime, y, dpp); +} + +/** + * Given an swift username, finds the user_info associated with it. + * returns: 0 on success, -ERR# on failure (including nonexistence) + */ +int RGWSI_User_RADOS::get_user_info_by_swift(RGWSI_MetaBackend::Context *ctx, + const string& swift_name, + RGWUserInfo *info, /* out */ + RGWObjVersionTracker * const objv_tracker, + real_time * const pmtime, optional_yield y, + const DoutPrefixProvider *dpp) +{ + return get_user_info_from_index(ctx, + swift_name, + svc.zone->get_zone_params().user_swift_pool, + info, objv_tracker, pmtime, y, dpp); +} + +/** + * Given an access key, finds the user info associated with it. + * returns: 0 on success, -ERR# on failure (including nonexistence) + */ +int RGWSI_User_RADOS::get_user_info_by_access_key(RGWSI_MetaBackend::Context *ctx, + const std::string& access_key, + RGWUserInfo *info, + RGWObjVersionTracker* objv_tracker, + real_time *pmtime, optional_yield y, + const DoutPrefixProvider *dpp) +{ + return get_user_info_from_index(ctx, + access_key, + svc.zone->get_zone_params().user_keys_pool, + info, objv_tracker, pmtime, y, dpp); +} + +int RGWSI_User_RADOS::cls_user_update_buckets(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, list<cls_user_bucket_entry>& entries, bool add, optional_yield y) +{ + auto rados_obj = svc.rados->obj(obj); + int r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + + librados::ObjectWriteOperation op; + cls_user_set_buckets(op, entries, add); + r = rados_obj.operate(dpp, &op, y); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_User_RADOS::cls_user_add_bucket(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, const cls_user_bucket_entry& entry, optional_yield y) +{ + list<cls_user_bucket_entry> l; + l.push_back(entry); + + return cls_user_update_buckets(dpp, obj, l, true, y); +} + +int RGWSI_User_RADOS::cls_user_remove_bucket(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, const cls_user_bucket& bucket, optional_yield y) +{ + auto rados_obj = svc.rados->obj(obj); + int r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + + librados::ObjectWriteOperation op; + ::cls_user_remove_bucket(op, bucket); + r = rados_obj.operate(dpp, &op, y); + if (r < 0) + return r; + + return 0; +} + +int RGWSI_User_RADOS::add_bucket(const DoutPrefixProvider *dpp, + const rgw_user& user, + const rgw_bucket& bucket, + ceph::real_time creation_time, + optional_yield y) +{ + int ret; + + cls_user_bucket_entry new_bucket; + + bucket.convert(&new_bucket.bucket); + new_bucket.size = 0; + if (real_clock::is_zero(creation_time)) + new_bucket.creation_time = real_clock::now(); + else + new_bucket.creation_time = creation_time; + + rgw_raw_obj obj = get_buckets_obj(user); + ret = cls_user_add_bucket(dpp, obj, new_bucket, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: error adding bucket to user: ret=" << ret << dendl; + return ret; + } + + return 0; +} + + +int RGWSI_User_RADOS::remove_bucket(const DoutPrefixProvider *dpp, + const rgw_user& user, + const rgw_bucket& _bucket, + optional_yield y) +{ + cls_user_bucket bucket; + bucket.name = _bucket.name; + rgw_raw_obj obj = get_buckets_obj(user); + int ret = cls_user_remove_bucket(dpp, obj, bucket, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: error removing bucket from user: ret=" << ret << dendl; + } + + return 0; +} + +int RGWSI_User_RADOS::cls_user_flush_bucket_stats(const DoutPrefixProvider *dpp, + rgw_raw_obj& user_obj, + const RGWBucketEnt& ent, optional_yield y) +{ + cls_user_bucket_entry entry; + ent.convert(&entry); + + list<cls_user_bucket_entry> entries; + entries.push_back(entry); + + int r = cls_user_update_buckets(dpp, user_obj, entries, false, y); + if (r < 0) { + ldpp_dout(dpp, 20) << "cls_user_update_buckets() returned " << r << dendl; + return r; + } + + return 0; +} + +int RGWSI_User_RADOS::cls_user_list_buckets(const DoutPrefixProvider *dpp, + rgw_raw_obj& obj, + const string& in_marker, + const string& end_marker, + const int max_entries, + list<cls_user_bucket_entry>& entries, + string * const out_marker, + bool * const truncated, + optional_yield y) +{ + auto rados_obj = svc.rados->obj(obj); + int r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + + librados::ObjectReadOperation op; + int rc; + + cls_user_bucket_list(op, in_marker, end_marker, max_entries, entries, out_marker, truncated, &rc); + bufferlist ibl; + r = rados_obj.operate(dpp, &op, &ibl, y); + if (r < 0) + return r; + if (rc < 0) + return rc; + + return 0; +} + +int RGWSI_User_RADOS::list_buckets(const DoutPrefixProvider *dpp, + const rgw_user& user, + const string& marker, + const string& end_marker, + uint64_t max, + RGWUserBuckets *buckets, + bool *is_truncated, optional_yield y) +{ + int ret; + + buckets->clear(); + if (user.id == RGW_USER_ANON_ID) { + ldpp_dout(dpp, 20) << "RGWSI_User_RADOS::list_buckets(): anonymous user" << dendl; + *is_truncated = false; + return 0; + } + rgw_raw_obj obj = get_buckets_obj(user); + + bool truncated = false; + string m = marker; + + uint64_t total = 0; + + do { + std::list<cls_user_bucket_entry> entries; + ret = cls_user_list_buckets(dpp, obj, m, end_marker, max - total, entries, &m, &truncated, y); + if (ret == -ENOENT) { + ret = 0; + } + + if (ret < 0) { + return ret; + } + + for (auto& entry : entries) { + buckets->add(RGWBucketEnt(user, std::move(entry))); + total++; + } + + } while (truncated && total < max); + + if (is_truncated) { + *is_truncated = truncated; + } + + return 0; +} + +int RGWSI_User_RADOS::flush_bucket_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, + const RGWBucketEnt& ent, + optional_yield y) +{ + rgw_raw_obj obj = get_buckets_obj(user); + + return cls_user_flush_bucket_stats(dpp, obj, ent, y); +} + +int RGWSI_User_RADOS::reset_bucket_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, + optional_yield y) +{ + return cls_user_reset_stats(dpp, user, y); +} + +int RGWSI_User_RADOS::cls_user_reset_stats(const DoutPrefixProvider *dpp, const rgw_user& user, optional_yield y) +{ + rgw_raw_obj obj = get_buckets_obj(user); + auto rados_obj = svc.rados->obj(obj); + int rval, r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + + cls_user_reset_stats2_op call; + cls_user_reset_stats2_ret ret; + + do { + buffer::list in, out; + librados::ObjectWriteOperation op; + + call.time = real_clock::now(); + ret.update_call(call); + + encode(call, in); + op.exec("user", "reset_user_stats2", in, &out, &rval); + r = rados_obj.operate(dpp, &op, y, librados::OPERATION_RETURNVEC); + if (r < 0) { + return r; + } + try { + auto bliter = out.cbegin(); + decode(ret, bliter); + } catch (ceph::buffer::error& err) { + return -EINVAL; + } + } while (ret.truncated); + + return rval; +} + +int RGWSI_User_RADOS::complete_flush_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, optional_yield y) +{ + rgw_raw_obj obj = get_buckets_obj(user); + auto rados_obj = svc.rados->obj(obj); + int r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + librados::ObjectWriteOperation op; + ::cls_user_complete_stats_sync(op); + return rados_obj.operate(dpp, &op, y); +} + +int RGWSI_User_RADOS::cls_user_get_header(const DoutPrefixProvider *dpp, + const rgw_user& user, cls_user_header *header, + optional_yield y) +{ + rgw_raw_obj obj = get_buckets_obj(user); + auto rados_obj = svc.rados->obj(obj); + int r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + int rc; + bufferlist ibl; + librados::ObjectReadOperation op; + ::cls_user_get_header(op, header, &rc); + return rados_obj.operate(dpp, &op, &ibl, y); +} + +int RGWSI_User_RADOS::cls_user_get_header_async(const DoutPrefixProvider *dpp, const string& user_str, RGWGetUserHeader_CB *cb) +{ + rgw_raw_obj obj = get_buckets_obj(rgw_user(user_str)); + auto rados_obj = svc.rados->obj(obj); + int r = rados_obj.open(dpp); + if (r < 0) { + return r; + } + + auto& ref = rados_obj.get_ref(); + + r = ::cls_user_get_header_async(ref.pool.ioctx(), ref.obj.oid, cb); + if (r < 0) { + return r; + } + + return 0; +} + +int RGWSI_User_RADOS::read_stats(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const rgw_user& user, RGWStorageStats *stats, + ceph::real_time *last_stats_sync, + ceph::real_time *last_stats_update, + optional_yield y) +{ + string user_str = user.to_str(); + + RGWUserInfo info; + real_time mtime; + int ret = read_user_info(ctx, user, &info, nullptr, &mtime, nullptr, nullptr, y, dpp); + if (ret < 0) + { + return ret; + } + + cls_user_header header; + int r = cls_user_get_header(dpp, rgw_user(user_str), &header, y); + if (r < 0 && r != -ENOENT) + return r; + + const cls_user_stats& hs = header.stats; + + stats->size = hs.total_bytes; + stats->size_rounded = hs.total_bytes_rounded; + stats->num_objects = hs.total_entries; + + if (last_stats_sync) { + *last_stats_sync = header.last_stats_sync; + } + + if (last_stats_update) { + *last_stats_update = header.last_stats_update; + } + + return 0; +} + +class RGWGetUserStatsContext : public RGWGetUserHeader_CB { + RGWGetUserStats_CB *cb; + +public: + explicit RGWGetUserStatsContext(RGWGetUserStats_CB * const cb) + : cb(cb) {} + + void handle_response(int r, cls_user_header& header) override { + const cls_user_stats& hs = header.stats; + if (r >= 0) { + RGWStorageStats stats; + + stats.size = hs.total_bytes; + stats.size_rounded = hs.total_bytes_rounded; + stats.num_objects = hs.total_entries; + + cb->set_response(stats); + } + + cb->handle_response(r); + + cb->put(); + } +}; + +int RGWSI_User_RADOS::read_stats_async(const DoutPrefixProvider *dpp, + const rgw_user& user, RGWGetUserStats_CB *_cb) +{ + string user_str = user.to_str(); + + RGWGetUserStatsContext *cb = new RGWGetUserStatsContext(_cb); + int r = cls_user_get_header_async(dpp, user_str, cb); + if (r < 0) { + delete cb; + return r; + } + + return 0; +} + diff --git a/src/rgw/services/svc_user_rados.h b/src/rgw/services/svc_user_rados.h new file mode 100644 index 000000000..177f720d6 --- /dev/null +++ b/src/rgw/services/svc_user_rados.h @@ -0,0 +1,211 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw_service.h" + +#include "svc_meta_be.h" +#include "svc_user.h" +#include "rgw_bucket.h" + +class RGWSI_RADOS; +class RGWSI_Zone; +class RGWSI_SysObj; +class RGWSI_SysObj_Cache; +class RGWSI_Meta; +class RGWSI_SyncModules; +class RGWSI_MetaBackend_Handler; + +struct rgw_cache_entry_info; + +class RGWGetUserHeader_CB; +class RGWGetUserStats_CB; + +template <class T> +class RGWChainedCacheImpl; + +class RGWSI_User_RADOS : public RGWSI_User +{ + friend class PutOperation; + + std::unique_ptr<RGWSI_MetaBackend::Module> be_module; + RGWSI_MetaBackend_Handler *be_handler; + + struct user_info_cache_entry { + RGWUserInfo info; + RGWObjVersionTracker objv_tracker; + real_time mtime; + }; + + using RGWChainedCacheImpl_user_info_cache_entry = RGWChainedCacheImpl<user_info_cache_entry>; + std::unique_ptr<RGWChainedCacheImpl_user_info_cache_entry> uinfo_cache; + + rgw_raw_obj get_buckets_obj(const rgw_user& user_id) const; + + int get_user_info_from_index(RGWSI_MetaBackend::Context *ctx, + const std::string& key, + const rgw_pool& pool, + RGWUserInfo *info, + RGWObjVersionTracker * const objv_tracker, + real_time * const pmtime, + optional_yield y, + const DoutPrefixProvider *dpp); + + int remove_uid_index(RGWSI_MetaBackend::Context *ctx, const RGWUserInfo& user_info, RGWObjVersionTracker *objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp); + + int remove_key_index(const DoutPrefixProvider *dpp, const RGWAccessKey& access_key, optional_yield y); + int remove_email_index(const DoutPrefixProvider *dpp, const std::string& email, optional_yield y); + int remove_swift_name_index(const DoutPrefixProvider *dpp, const std::string& swift_name, optional_yield y); + + /* admin management */ + int cls_user_update_buckets(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, std::list<cls_user_bucket_entry>& entries, bool add, optional_yield y); + int cls_user_add_bucket(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, const cls_user_bucket_entry& entry, optional_yield y); + int cls_user_remove_bucket(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, const cls_user_bucket& bucket, optional_yield y); + + /* quota stats */ + int cls_user_flush_bucket_stats(const DoutPrefixProvider *dpp, rgw_raw_obj& user_obj, + const RGWBucketEnt& ent, optional_yield y); + int cls_user_list_buckets(const DoutPrefixProvider *dpp, + rgw_raw_obj& obj, + const std::string& in_marker, + const std::string& end_marker, + const int max_entries, + std::list<cls_user_bucket_entry>& entries, + std::string * const out_marker, + bool * const truncated, + optional_yield y); + + int cls_user_reset_stats(const DoutPrefixProvider *dpp, const rgw_user& user, optional_yield y); + int cls_user_get_header(const DoutPrefixProvider *dpp, const rgw_user& user, cls_user_header *header, optional_yield y); + int cls_user_get_header_async(const DoutPrefixProvider *dpp, const std::string& user, RGWGetUserHeader_CB *cb); + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; +public: + struct Svc { + RGWSI_User_RADOS *user{nullptr}; + RGWSI_RADOS *rados{nullptr}; + RGWSI_Zone *zone{nullptr}; + RGWSI_SysObj *sysobj{nullptr}; + RGWSI_SysObj_Cache *cache{nullptr}; + RGWSI_Meta *meta{nullptr}; + RGWSI_MetaBackend *meta_be{nullptr}; + RGWSI_SyncModules *sync_modules{nullptr}; + } svc; + + RGWSI_User_RADOS(CephContext *cct); + ~RGWSI_User_RADOS(); + + void init(RGWSI_RADOS *_rados_svc, + RGWSI_Zone *_zone_svc, RGWSI_SysObj *_sysobj_svc, + RGWSI_SysObj_Cache *_cache_svc, RGWSI_Meta *_meta_svc, + RGWSI_MetaBackend *_meta_be_svc, + RGWSI_SyncModules *_sync_modules); + + RGWSI_MetaBackend_Handler *get_be_handler() override { + return be_handler; + } + + int read_user_info(RGWSI_MetaBackend::Context *ctx, + const rgw_user& user, + RGWUserInfo *info, + RGWObjVersionTracker * const objv_tracker, + real_time * const pmtime, + rgw_cache_entry_info * const cache_info, + std::map<std::string, bufferlist> * const pattrs, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int store_user_info(RGWSI_MetaBackend::Context *ctx, + const RGWUserInfo& info, + RGWUserInfo *old_info, + RGWObjVersionTracker *objv_tracker, + const real_time& mtime, + bool exclusive, + std::map<std::string, bufferlist> *attrs, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int remove_user_info(RGWSI_MetaBackend::Context *ctx, + const RGWUserInfo& info, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + int get_user_info_by_email(RGWSI_MetaBackend::Context *ctx, + const std::string& email, RGWUserInfo *info, + RGWObjVersionTracker *objv_tracker, + real_time *pmtime, + optional_yield y, + const DoutPrefixProvider *dpp) override; + int get_user_info_by_swift(RGWSI_MetaBackend::Context *ctx, + const std::string& swift_name, + RGWUserInfo *info, /* out */ + RGWObjVersionTracker * const objv_tracker, + real_time * const pmtime, + optional_yield y, + const DoutPrefixProvider *dpp) override; + int get_user_info_by_access_key(RGWSI_MetaBackend::Context *ctx, + const std::string& access_key, + RGWUserInfo *info, + RGWObjVersionTracker* objv_tracker, + real_time *pmtime, + optional_yield y, + const DoutPrefixProvider *dpp) override; + + /* user buckets directory */ + + int add_bucket(const DoutPrefixProvider *dpp, + const rgw_user& user, + const rgw_bucket& bucket, + ceph::real_time creation_time, + optional_yield y) override; + int remove_bucket(const DoutPrefixProvider *dpp, + const rgw_user& user, + const rgw_bucket& _bucket, + optional_yield y) override; + int list_buckets(const DoutPrefixProvider *dpp, + const rgw_user& user, + const std::string& marker, + const std::string& end_marker, + uint64_t max, + RGWUserBuckets *buckets, + bool *is_truncated, + optional_yield y) override; + + /* quota related */ + int flush_bucket_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, + const RGWBucketEnt& ent, optional_yield y) override; + + int complete_flush_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, optional_yield y) override; + + int reset_bucket_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, + optional_yield y) override; + int read_stats(const DoutPrefixProvider *dpp, + RGWSI_MetaBackend::Context *ctx, + const rgw_user& user, RGWStorageStats *stats, + ceph::real_time *last_stats_sync, /* last time a full stats sync completed */ + ceph::real_time *last_stats_update, + optional_yield y) override; /* last time a stats update was done */ + + int read_stats_async(const DoutPrefixProvider *dpp, const rgw_user& user, + RGWGetUserStats_CB *cb) override; +}; + diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc new file mode 100644 index 000000000..180d93712 --- /dev/null +++ b/src/rgw/services/svc_zone.cc @@ -0,0 +1,1100 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_zone.h" +#include "svc_rados.h" +#include "svc_sys_obj.h" +#include "svc_sync_modules.h" + +#include "rgw_zone.h" +#include "rgw_rest_conn.h" +#include "rgw_bucket_sync.h" + +#include "common/errno.h" +#include "include/random.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; +using namespace rgw_zone_defaults; + +RGWSI_Zone::RGWSI_Zone(CephContext *cct) : RGWServiceInstance(cct) +{ +} + +void RGWSI_Zone::init(RGWSI_SysObj *_sysobj_svc, + RGWSI_RADOS * _rados_svc, + RGWSI_SyncModules * _sync_modules_svc, + RGWSI_Bucket_Sync *_bucket_sync_svc) +{ + sysobj_svc = _sysobj_svc; + rados_svc = _rados_svc; + sync_modules_svc = _sync_modules_svc; + bucket_sync_svc = _bucket_sync_svc; + + realm = new RGWRealm(); + zonegroup = new RGWZoneGroup(); + zone_public_config = new RGWZone(); + zone_params = new RGWZoneParams(); + current_period = new RGWPeriod(); +} + +RGWSI_Zone::~RGWSI_Zone() +{ + delete realm; + delete zonegroup; + delete zone_public_config; + delete zone_params; + delete current_period; +} + +std::shared_ptr<RGWBucketSyncPolicyHandler> RGWSI_Zone::get_sync_policy_handler(std::optional<rgw_zone_id> zone) const { + if (!zone || *zone == zone_id()) { + return sync_policy_handler; + } + auto iter = sync_policy_handlers.find(*zone); + if (iter == sync_policy_handlers.end()) { + return std::shared_ptr<RGWBucketSyncPolicyHandler>(); + } + return iter->second; +} + +bool RGWSI_Zone::zone_syncs_from(const RGWZone& target_zone, const RGWZone& source_zone) const +{ + return target_zone.syncs_from(source_zone.name) && + sync_modules_svc->get_manager()->supports_data_export(source_zone.tier_type); +} + +bool RGWSI_Zone::zone_syncs_from(const RGWZone& source_zone) const +{ + auto target_zone = get_zone(); + bool found = false; + + for (auto s : data_sync_source_zones) { + if (s->id == source_zone.id) { + found = true; + break; + } + } + return found && target_zone.syncs_from(source_zone.name) && + sync_modules_svc->get_manager()->supports_data_export(source_zone.tier_type); +} + +int RGWSI_Zone::search_realm_with_zone(const DoutPrefixProvider *dpp, + const rgw_zone_id& zid, + RGWRealm *prealm, + RGWPeriod *pperiod, + RGWZoneGroup *pzonegroup, + bool *pfound, + optional_yield y) +{ + auto& found = *pfound; + + found = false; + + list<string> realms; + int r = list_realms(dpp, realms); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to list realms: r=" << r << dendl; + return r; + } + + for (auto& realm_name : realms) { + string realm_id; + RGWRealm realm(realm_id, realm_name); + r = realm.init(dpp, cct, sysobj_svc, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: can't open realm " << realm_name << ": " << cpp_strerror(-r) << " ... skipping" << dendl; + continue; + } + + r = realm.find_zone(dpp, zid, pperiod, + pzonegroup, &found, y); + if (r < 0) { + ldpp_dout(dpp, 20) << __func__ << "(): ERROR: realm.find_zone() returned r=" << r<< dendl; + return r; + } + + if (found) { + *prealm = realm; + ldpp_dout(dpp, 20) << __func__ << "(): found realm_id=" << realm_id << " realm_name=" << realm_name << dendl; + return 0; + } + } + + return 0; +} + +int RGWSI_Zone::do_start(optional_yield y, const DoutPrefixProvider *dpp) +{ + int ret = sysobj_svc->start(y, dpp); + if (ret < 0) { + return ret; + } + + assert(sysobj_svc->is_started()); /* if not then there's ordering issue */ + + ret = rados_svc->start(y, dpp); + if (ret < 0) { + return ret; + } + + ret = realm->init(dpp, cct, sysobj_svc, y); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "failed reading realm info: ret "<< ret << " " << cpp_strerror(-ret) << dendl; + return ret; + } + + ldpp_dout(dpp, 20) << "realm " << realm->get_name() << " " << realm->get_id() << dendl; + ret = current_period->init(dpp, cct, sysobj_svc, realm->get_id(), y, + realm->get_name()); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "failed reading current period info: " << " " << cpp_strerror(-ret) << dendl; + return ret; + } + + ret = zone_params->init(dpp, cct, sysobj_svc, y); + bool found_zone = (ret == 0); + if (ret < 0 && ret != -ENOENT) { + lderr(cct) << "failed reading zone info: ret "<< ret << " " << cpp_strerror(-ret) << dendl; + return ret; + } + + cur_zone_id = rgw_zone_id(zone_params->get_id()); + + bool found_period_conf = false; + + /* try to find zone in period config (if we have one) */ + if (found_zone && + !current_period->get_id().empty()) { + found_period_conf = current_period->find_zone(dpp, + cur_zone_id, + zonegroup, + y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: current_period->find_zone() returned ret=" << ret << dendl; + return ret; + } + if (!found_period_conf) { + ldpp_dout(dpp, 0) << "period (" << current_period->get_id() << " does not have zone " << cur_zone_id << " configured" << dendl; + } + } + + RGWRealm search_realm; + + if (found_zone && + !found_period_conf) { + ldpp_dout(dpp, 20) << "searching for the correct realm" << dendl; + ret = search_realm_with_zone(dpp, + cur_zone_id, + realm, + current_period, + zonegroup, + &found_period_conf, + y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: search_realm_conf() failed: ret="<< ret << dendl; + return ret; + } + } + bool zg_initialized = found_period_conf; + + if (!zg_initialized) { + /* couldn't find a proper period config, use local zonegroup */ + ret = zonegroup->init(dpp, cct, sysobj_svc, y); + zg_initialized = (ret == 0); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "failed reading zonegroup info: " << cpp_strerror(-ret) << dendl; + return ret; + } + } + + auto& zonegroup_param = cct->_conf->rgw_zonegroup; + bool init_from_period = found_period_conf; + bool explicit_zg = !zonegroup_param.empty(); + + if (!zg_initialized && + (!explicit_zg || zonegroup_param == default_zonegroup_name)) { + /* we couldn't initialize any zonegroup, + falling back to a non-multisite config with default zonegroup */ + ret = create_default_zg(dpp, y); + if (ret < 0) { + return ret; + } + zg_initialized = true; + } + + if (!zg_initialized) { + ldpp_dout(dpp, 0) << "ERROR: could not find zonegroup (" << zonegroup_param << ")" << dendl; + return -ENOENT; + } + + /* we have zonegroup now */ + + if (explicit_zg && + zonegroup->get_name() != zonegroup_param) { + ldpp_dout(dpp, 0) << "ERROR: incorrect zonegroup: " << zonegroup_param << " (got: " << zonegroup_param << ", expected: " << zonegroup->get_name() << ")" << dendl; + return -EINVAL; + } + + auto& zone_param = cct->_conf->rgw_zone; + bool explicit_zone = !zone_param.empty(); + + if (!found_zone) { + if ((!explicit_zone || zone_param == default_zone_name) && + zonegroup->get_name() == default_zonegroup_name) { + ret = init_default_zone(dpp, y); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + cur_zone_id = zone_params->get_id(); + } else { + ldpp_dout(dpp, 0) << "ERROR: could not find zone (" << zone_param << ")" << dendl; + return -ENOENT; + } + } + + /* we have zone now */ + + auto zone_iter = zonegroup->zones.find(zone_params->get_id()); + if (zone_iter == zonegroup->zones.end()) { + /* shouldn't happen if relying on period config */ + if (!init_from_period) { + ldpp_dout(dpp, -1) << "Cannot find zone id=" << zone_params->get_id() << " (name=" << zone_params->get_name() << ")" << dendl; + return -EINVAL; + } + ldpp_dout(dpp, 1) << "Cannot find zone id=" << zone_params->get_id() << " (name=" << zone_params->get_name() << "), switching to local zonegroup configuration" << dendl; + init_from_period = false; + zone_iter = zonegroup->zones.find(zone_params->get_id()); + } + if (zone_iter == zonegroup->zones.end()) { + ldpp_dout(dpp, -1) << "Cannot find zone id=" << zone_params->get_id() << " (name=" << zone_params->get_name() << ")" << dendl; + return -EINVAL; + } + *zone_public_config = zone_iter->second; + ldout(cct, 20) << "zone " << zone_params->get_name() << " found" << dendl; + + ldpp_dout(dpp, 4) << "Realm: " << std::left << setw(20) << realm->get_name() << " (" << realm->get_id() << ")" << dendl; + ldpp_dout(dpp, 4) << "ZoneGroup: " << std::left << setw(20) << zonegroup->get_name() << " (" << zonegroup->get_id() << ")" << dendl; + ldpp_dout(dpp, 4) << "Zone: " << std::left << setw(20) << zone_params->get_name() << " (" << zone_params->get_id() << ")" << dendl; + + if (init_from_period) { + ldpp_dout(dpp, 4) << "using period configuration: " << current_period->get_id() << ":" << current_period->get_epoch() << dendl; + ret = init_zg_from_period(dpp, y); + if (ret < 0) { + return ret; + } + } else { + ldout(cct, 10) << "cannot find current period zonegroup using local zonegroup configuration" << dendl; + ret = init_zg_from_local(dpp, y); + if (ret < 0) { + return ret; + } + // read period_config into current_period + auto& period_config = current_period->get_config(); + ret = period_config.read(dpp, sysobj_svc, zonegroup->realm_id, y); + if (ret < 0 && ret != -ENOENT) { + ldout(cct, 0) << "ERROR: failed to read period config: " + << cpp_strerror(ret) << dendl; + return ret; + } + } + + zone_short_id = current_period->get_map().get_zone_short_id(zone_params->get_id()); + + for (auto ziter : zonegroup->zones) { + auto zone_handler = std::make_shared<RGWBucketSyncPolicyHandler>(this, sync_modules_svc, bucket_sync_svc, ziter.second.id); + ret = zone_handler->init(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR: could not initialize zone policy handler for zone=" << ziter.second.name << dendl; + return ret; + } + sync_policy_handlers[ziter.second.id] = zone_handler; + } + + sync_policy_handler = sync_policy_handlers[zone_id()]; /* we made sure earlier that zonegroup->zones has our zone */ + + set<rgw_zone_id> source_zones; + set<rgw_zone_id> target_zones; + + sync_policy_handler->reflect(dpp, nullptr, nullptr, + nullptr, nullptr, + &source_zones, + &target_zones, + false); /* relaxed: also get all zones that we allow to sync to/from */ + + ret = sync_modules_svc->start(y, dpp); + if (ret < 0) { + return ret; + } + + auto sync_modules = sync_modules_svc->get_manager(); + RGWSyncModuleRef sm; + if (!sync_modules->get_module(zone_public_config->tier_type, &sm)) { + ldpp_dout(dpp, -1) << "ERROR: tier type not found: " << zone_public_config->tier_type << dendl; + return -EINVAL; + } + + writeable_zone = sm->supports_writes(); + exports_data = sm->supports_data_export(); + + /* first build all zones index */ + for (auto ziter : zonegroup->zones) { + const rgw_zone_id& id = ziter.first; + RGWZone& z = ziter.second; + zone_id_by_name[z.name] = id; + zone_by_id[id] = z; + } + + if (zone_by_id.find(zone_id()) == zone_by_id.end()) { + ldpp_dout(dpp, 0) << "WARNING: could not find zone config in zonegroup for local zone (" << zone_id() << "), will use defaults" << dendl; + } + + for (const auto& ziter : zonegroup->zones) { + const rgw_zone_id& id = ziter.first; + const RGWZone& z = ziter.second; + if (id == zone_id()) { + continue; + } + if (z.endpoints.empty()) { + ldpp_dout(dpp, 0) << "WARNING: can't generate connection for zone " << z.id << " id " << z.name << ": no endpoints defined" << dendl; + continue; + } + ldpp_dout(dpp, 20) << "generating connection object for zone " << z.name << " id " << z.id << dendl; + RGWRESTConn *conn = new RGWRESTConn(cct, z.id, z.endpoints, zone_params->system_key, zonegroup->get_id(), zonegroup->api_name); + zone_conn_map[id] = conn; + + bool zone_is_source = source_zones.find(z.id) != source_zones.end(); + bool zone_is_target = target_zones.find(z.id) != target_zones.end(); + + if (zone_is_source || zone_is_target) { + if (zone_is_source && sync_modules->supports_data_export(z.tier_type)) { + data_sync_source_zones.push_back(&z); + } + if (zone_is_target) { + zone_data_notify_to_map[id] = conn; + } + } else { + ldpp_dout(dpp, 20) << "NOTICE: not syncing to/from zone " << z.name << " id " << z.id << dendl; + } + } + + ldpp_dout(dpp, 20) << "started zone id=" << zone_params->get_id() << " (name=" << zone_params->get_name() << + ") with tier type = " << zone_public_config->tier_type << dendl; + + return 0; +} + +void RGWSI_Zone::shutdown() +{ + delete rest_master_conn; + + for (auto& item : zone_conn_map) { + auto conn = item.second; + delete conn; + } + + for (auto& item : zonegroup_conn_map) { + auto conn = item.second; + delete conn; + } +} + +int RGWSI_Zone::list_regions(const DoutPrefixProvider *dpp, list<string>& regions) +{ + RGWZoneGroup zonegroup; + RGWSI_SysObj::Pool syspool = sysobj_svc->get_pool(zonegroup.get_pool(cct)); + + return syspool.list_prefixed_objs(dpp, region_info_oid_prefix, ®ions); +} + +int RGWSI_Zone::list_zonegroups(const DoutPrefixProvider *dpp, list<string>& zonegroups) +{ + RGWZoneGroup zonegroup; + RGWSI_SysObj::Pool syspool = sysobj_svc->get_pool(zonegroup.get_pool(cct)); + + return syspool.list_prefixed_objs(dpp, zonegroup_names_oid_prefix, &zonegroups); +} + +int RGWSI_Zone::list_zones(const DoutPrefixProvider *dpp, list<string>& zones) +{ + RGWZoneParams zoneparams; + RGWSI_SysObj::Pool syspool = sysobj_svc->get_pool(zoneparams.get_pool(cct)); + + return syspool.list_prefixed_objs(dpp, zone_names_oid_prefix, &zones); +} + +int RGWSI_Zone::list_realms(const DoutPrefixProvider *dpp, list<string>& realms) +{ + RGWRealm realm(cct, sysobj_svc); + RGWSI_SysObj::Pool syspool = sysobj_svc->get_pool(realm.get_pool(cct)); + + return syspool.list_prefixed_objs(dpp, realm_names_oid_prefix, &realms); +} + +int RGWSI_Zone::list_periods(const DoutPrefixProvider *dpp, list<string>& periods) +{ + RGWPeriod period; + list<string> raw_periods; + RGWSI_SysObj::Pool syspool = sysobj_svc->get_pool(period.get_pool(cct)); + int ret = syspool.list_prefixed_objs(dpp, period.get_info_oid_prefix(), &raw_periods); + if (ret < 0) { + return ret; + } + for (const auto& oid : raw_periods) { + size_t pos = oid.find("."); + if (pos != std::string::npos) { + periods.push_back(oid.substr(0, pos)); + } else { + periods.push_back(oid); + } + } + periods.sort(); // unique() only detects duplicates if they're adjacent + periods.unique(); + return 0; +} + + +int RGWSI_Zone::list_periods(const DoutPrefixProvider *dpp, const string& current_period, list<string>& periods, optional_yield y) +{ + int ret = 0; + string period_id = current_period; + while(!period_id.empty()) { + RGWPeriod period(period_id); + ret = period.init(dpp, cct, sysobj_svc, y); + if (ret < 0) { + return ret; + } + periods.push_back(period.get_id()); + period_id = period.get_predecessor(); + } + + return ret; +} + +/** + * Add new connection to connections map + * @param zonegroup_conn_map map which new connection will be added to + * @param zonegroup zonegroup which new connection will connect to + * @param new_connection pointer to new connection instance + */ +static void add_new_connection_to_map(map<string, RGWRESTConn *> &zonegroup_conn_map, + const RGWZoneGroup &zonegroup, RGWRESTConn *new_connection) +{ + // Delete if connection is already exists + map<string, RGWRESTConn *>::iterator iterZoneGroup = zonegroup_conn_map.find(zonegroup.get_id()); + if (iterZoneGroup != zonegroup_conn_map.end()) { + delete iterZoneGroup->second; + } + + // Add new connection to connections map + zonegroup_conn_map[zonegroup.get_id()] = new_connection; +} + +int RGWSI_Zone::init_zg_from_period(const DoutPrefixProvider *dpp, optional_yield y) +{ + ldout(cct, 20) << "period zonegroup name " << zonegroup->get_name() << dendl; + + map<string, RGWZoneGroup>::const_iterator iter = + current_period->get_map().zonegroups.find(zonegroup->get_id()); + + if (iter != current_period->get_map().zonegroups.end()) { + ldpp_dout(dpp, 20) << "using current period zonegroup " << zonegroup->get_name() << dendl; + *zonegroup = iter->second; + int ret = zonegroup->init(dpp, cct, sysobj_svc, y, false); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed init zonegroup: " << " " << cpp_strerror(-ret) << dendl; + return ret; + } + } + for (iter = current_period->get_map().zonegroups.begin(); + iter != current_period->get_map().zonegroups.end(); ++iter){ + const RGWZoneGroup& zg = iter->second; + // use endpoints from the zonegroup's master zone + auto master = zg.zones.find(zg.master_zone); + if (master == zg.zones.end()) { + // Check for empty zonegroup which can happen if zone was deleted before removal + if (zg.zones.size() == 0) + continue; + // fix missing master zone for a single zone zonegroup + if (zg.master_zone.empty() && zg.zones.size() == 1) { + master = zg.zones.begin(); + ldpp_dout(dpp, 0) << "zonegroup " << zg.get_name() << " missing master_zone, setting zone " << + master->second.name << " id:" << master->second.id << " as master" << dendl; + if (zonegroup->get_id() == zg.get_id()) { + zonegroup->master_zone = master->second.id; + int ret = zonegroup->update(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "error updating zonegroup : " << cpp_strerror(-ret) << dendl; + return ret; + } + } else { + RGWZoneGroup fixed_zg(zg.get_id(),zg.get_name()); + int ret = fixed_zg.init(dpp, cct, sysobj_svc, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "error initializing zonegroup : " << cpp_strerror(-ret) << dendl; + return ret; + } + fixed_zg.master_zone = master->second.id; + ret = fixed_zg.update(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "error initializing zonegroup : " << cpp_strerror(-ret) << dendl; + return ret; + } + } + } else { + ldpp_dout(dpp, 0) << "zonegroup " << zg.get_name() << " missing zone for master_zone=" << + zg.master_zone << dendl; + return -EINVAL; + } + } + const auto& endpoints = master->second.endpoints; + add_new_connection_to_map(zonegroup_conn_map, zg, new RGWRESTConn(cct, zg.get_id(), endpoints, zone_params->system_key, zonegroup->get_id(), zg.api_name)); + if (!current_period->get_master_zonegroup().empty() && + zg.get_id() == current_period->get_master_zonegroup()) { + rest_master_conn = new RGWRESTConn(cct, zg.get_id(), endpoints, zone_params->system_key, zonegroup->get_id(), zg.api_name); + } + } + + return 0; +} + +int RGWSI_Zone::create_default_zg(const DoutPrefixProvider *dpp, optional_yield y) +{ + ldout(cct, 10) << "Creating default zonegroup " << dendl; + int ret = zonegroup->create_default(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failure in zonegroup create_default: ret "<< ret << " " << cpp_strerror(-ret) + << dendl; + return ret; + } + ret = zonegroup->init(dpp, cct, sysobj_svc, y); + if (ret < 0) { + ldout(cct, 0) << "failure in zonegroup create_default: ret "<< ret << " " << cpp_strerror(-ret) + << dendl; + return ret; + } + + return 0; +} + +int RGWSI_Zone::init_default_zone(const DoutPrefixProvider *dpp, optional_yield y) +{ + ldpp_dout(dpp, 10) << " Using default name "<< default_zone_name << dendl; + zone_params->set_name(default_zone_name); + int ret = zone_params->init(dpp, cct, sysobj_svc, y); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "failed reading zone params info: " << " " << cpp_strerror(-ret) << dendl; + return ret; + } + + return 0; +} + +int RGWSI_Zone::init_zg_from_local(const DoutPrefixProvider *dpp, optional_yield y) +{ + ldpp_dout(dpp, 20) << "zonegroup " << zonegroup->get_name() << dendl; + if (zonegroup->is_master_zonegroup()) { + // use endpoints from the zonegroup's master zone + auto master = zonegroup->zones.find(zonegroup->master_zone); + if (master == zonegroup->zones.end()) { + // fix missing master zone for a single zone zonegroup + if (zonegroup->master_zone.empty() && zonegroup->zones.size() == 1) { + master = zonegroup->zones.begin(); + ldpp_dout(dpp, 0) << "zonegroup " << zonegroup->get_name() << " missing master_zone, setting zone " << + master->second.name << " id:" << master->second.id << " as master" << dendl; + zonegroup->master_zone = master->second.id; + int ret = zonegroup->update(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "error initializing zonegroup : " << cpp_strerror(-ret) << dendl; + return ret; + } + } else { + ldpp_dout(dpp, 0) << "zonegroup " << zonegroup->get_name() << " missing zone for " + "master_zone=" << zonegroup->master_zone << dendl; + return -EINVAL; + } + } + const auto& endpoints = master->second.endpoints; + rest_master_conn = new RGWRESTConn(cct, zonegroup->get_id(), endpoints, zone_params->system_key, zonegroup->get_id(), zonegroup->api_name); + } + + return 0; +} + +const RGWZoneParams& RGWSI_Zone::get_zone_params() const +{ + return *zone_params; +} + +const RGWZone& RGWSI_Zone::get_zone() const +{ + return *zone_public_config; +} + +const RGWZoneGroup& RGWSI_Zone::get_zonegroup() const +{ + return *zonegroup; +} + +int RGWSI_Zone::get_zonegroup(const string& id, RGWZoneGroup& zg) const +{ + int ret = 0; + if (id == zonegroup->get_id()) { + zg = *zonegroup; + } else if (!current_period->get_id().empty()) { + ret = current_period->get_zonegroup(zg, id); + } + return ret; +} + +const RGWRealm& RGWSI_Zone::get_realm() const +{ + return *realm; +} + +const RGWPeriod& RGWSI_Zone::get_current_period() const +{ + return *current_period; +} + +const string& RGWSI_Zone::get_current_period_id() const +{ + return current_period->get_id(); +} + +bool RGWSI_Zone::has_zonegroup_api(const std::string& api) const +{ + if (!current_period->get_id().empty()) { + const auto& zonegroups_by_api = current_period->get_map().zonegroups_by_api; + if (zonegroups_by_api.find(api) != zonegroups_by_api.end()) + return true; + } else if (zonegroup->api_name == api) { + return true; + } + return false; +} + +bool RGWSI_Zone::zone_is_writeable() +{ + return writeable_zone && !get_zone().is_read_only(); +} + +uint32_t RGWSI_Zone::get_zone_short_id() const +{ + return zone_short_id; +} + +const string& RGWSI_Zone::zone_name() const +{ + return get_zone_params().get_name(); +} + +RGWZone* RGWSI_Zone::find_zone(const rgw_zone_id& id) +{ + auto iter = zone_by_id.find(id); + if (iter == zone_by_id.end()) { + return nullptr; + } + return &(iter->second); +} + +RGWRESTConn *RGWSI_Zone::get_zone_conn(const rgw_zone_id& zone_id) { + auto citer = zone_conn_map.find(zone_id.id); + if (citer == zone_conn_map.end()) { + return NULL; + } + + return citer->second; +} + +RGWRESTConn *RGWSI_Zone::get_zone_conn_by_name(const string& name) { + auto i = zone_id_by_name.find(name); + if (i == zone_id_by_name.end()) { + return NULL; + } + + return get_zone_conn(i->second); +} + +bool RGWSI_Zone::find_zone_id_by_name(const string& name, rgw_zone_id *id) { + auto i = zone_id_by_name.find(name); + if (i == zone_id_by_name.end()) { + return false; + } + *id = i->second; + return true; +} + +bool RGWSI_Zone::need_to_sync() const +{ + return !(zonegroup->master_zone.empty() || + !rest_master_conn || + current_period->get_id().empty()); +} + +bool RGWSI_Zone::need_to_log_data() const +{ + return (zone_public_config->log_data && sync_module_exports_data()); +} + +bool RGWSI_Zone::is_meta_master() const +{ + if (!zonegroup->is_master_zonegroup()) { + return false; + } + + return (zonegroup->master_zone == zone_public_config->id); +} + +bool RGWSI_Zone::need_to_log_metadata() const +{ + return is_meta_master() && + (zonegroup->zones.size() > 1 || current_period->is_multi_zonegroups_with_zones()); +} + +bool RGWSI_Zone::can_reshard() const +{ + if (current_period->get_id().empty()) { + return true; // no realm + } + if (zonegroup->zones.size() == 1 && current_period->is_single_zonegroup()) { + return true; // single zone/zonegroup + } + // 'resharding' feature enabled in zonegroup + return zonegroup->supports(rgw::zone_features::resharding); +} + +/** + * Check to see if the bucket metadata could be synced + * bucket: the bucket to check + * Returns false is the bucket is not synced + */ +bool RGWSI_Zone::is_syncing_bucket_meta(const rgw_bucket& bucket) +{ + + /* no current period */ + if (current_period->get_id().empty()) { + return false; + } + + /* zonegroup is not master zonegroup */ + if (!zonegroup->is_master_zonegroup()) { + return false; + } + + /* single zonegroup and a single zone */ + if (current_period->is_single_zonegroup() && zonegroup->zones.size() == 1) { + return false; + } + + /* zone is not master */ + if (zonegroup->master_zone != zone_public_config->id) { + return false; + } + + return true; +} + + +int RGWSI_Zone::select_new_bucket_location(const DoutPrefixProvider *dpp, const RGWUserInfo& user_info, const string& zonegroup_id, + const rgw_placement_rule& request_rule, + rgw_placement_rule *pselected_rule_name, RGWZonePlacementInfo *rule_info, + optional_yield y) +{ + /* first check that zonegroup exists within current period. */ + RGWZoneGroup zonegroup; + int ret = get_zonegroup(zonegroup_id, zonegroup); + if (ret < 0) { + ldpp_dout(dpp, 0) << "could not find zonegroup " << zonegroup_id << " in current period" << dendl; + return ret; + } + + const rgw_placement_rule *used_rule; + + /* find placement rule. Hierarchy: request rule > user default rule > zonegroup default rule */ + std::map<std::string, RGWZoneGroupPlacementTarget>::const_iterator titer; + + if (!request_rule.name.empty()) { + used_rule = &request_rule; + titer = zonegroup.placement_targets.find(request_rule.name); + if (titer == zonegroup.placement_targets.end()) { + ldpp_dout(dpp, 0) << "could not find requested placement id " << request_rule + << " within zonegroup " << dendl; + return -ERR_INVALID_LOCATION_CONSTRAINT; + } + } else if (!user_info.default_placement.name.empty()) { + used_rule = &user_info.default_placement; + titer = zonegroup.placement_targets.find(user_info.default_placement.name); + if (titer == zonegroup.placement_targets.end()) { + ldpp_dout(dpp, 0) << "could not find user default placement id " << user_info.default_placement + << " within zonegroup " << dendl; + return -ERR_INVALID_LOCATION_CONSTRAINT; + } + } else { + if (zonegroup.default_placement.name.empty()) { // zonegroup default rule as fallback, it should not be empty. + ldpp_dout(dpp, 0) << "misconfiguration, zonegroup default placement id should not be empty." << dendl; + return -ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION; + } else { + used_rule = &zonegroup.default_placement; + titer = zonegroup.placement_targets.find(zonegroup.default_placement.name); + if (titer == zonegroup.placement_targets.end()) { + ldpp_dout(dpp, 0) << "could not find zonegroup default placement id " << zonegroup.default_placement + << " within zonegroup " << dendl; + return -ERR_INVALID_LOCATION_CONSTRAINT; + } + } + } + + /* now check tag for the rule, whether user is permitted to use rule */ + const auto& target_rule = titer->second; + if (!target_rule.user_permitted(user_info.placement_tags)) { + ldpp_dout(dpp, 0) << "user not permitted to use placement rule " << titer->first << dendl; + return -EPERM; + } + + const string *storage_class = &request_rule.storage_class; + + if (storage_class->empty()) { + storage_class = &used_rule->storage_class; + } + + rgw_placement_rule rule(titer->first, *storage_class); + + if (pselected_rule_name) { + *pselected_rule_name = rule; + } + + return select_bucket_location_by_rule(dpp, rule, rule_info, y); +} + +int RGWSI_Zone::select_bucket_location_by_rule(const DoutPrefixProvider *dpp, const rgw_placement_rule& location_rule, RGWZonePlacementInfo *rule_info, optional_yield y) +{ + if (location_rule.name.empty()) { + /* we can only reach here if we're trying to set a bucket location from a bucket + * created on a different zone, using a legacy / default pool configuration + */ + if (rule_info) { + return select_legacy_bucket_placement(dpp, rule_info, y); + } + + return 0; + } + + /* + * make sure that zone has this rule configured. We're + * checking it for the local zone, because that's where this bucket object is going to + * reside. + */ + auto piter = zone_params->placement_pools.find(location_rule.name); + if (piter == zone_params->placement_pools.end()) { + /* couldn't find, means we cannot really place data for this bucket in this zone */ + ldpp_dout(dpp, 0) << "ERROR: This zone does not contain placement rule " + << location_rule << " present in the zonegroup!" << dendl; + return -EINVAL; + } + + auto storage_class = location_rule.get_storage_class(); + if (!piter->second.storage_class_exists(storage_class)) { + ldpp_dout(dpp, 5) << "requested storage class does not exist: " << storage_class << dendl; + return -EINVAL; + } + + + RGWZonePlacementInfo& placement_info = piter->second; + + if (rule_info) { + *rule_info = placement_info; + } + + return 0; +} + +int RGWSI_Zone::select_bucket_placement(const DoutPrefixProvider *dpp, const RGWUserInfo& user_info, const string& zonegroup_id, + const rgw_placement_rule& placement_rule, + rgw_placement_rule *pselected_rule, RGWZonePlacementInfo *rule_info, + optional_yield y) +{ + if (!zone_params->placement_pools.empty()) { + return select_new_bucket_location(dpp, user_info, zonegroup_id, placement_rule, + pselected_rule, rule_info, y); + } + + if (pselected_rule) { + pselected_rule->clear(); + } + + if (rule_info) { + return select_legacy_bucket_placement(dpp, rule_info, y); + } + + return 0; +} + +int RGWSI_Zone::select_legacy_bucket_placement(const DoutPrefixProvider *dpp, RGWZonePlacementInfo *rule_info, + optional_yield y) +{ + bufferlist map_bl; + map<string, bufferlist> m; + string pool_name; + bool write_map = false; + + rgw_raw_obj obj(zone_params->domain_root, avail_pools); + + auto sysobj = sysobj_svc->get_obj(obj); + int ret = sysobj.rop().read(dpp, &map_bl, y); + if (ret < 0) { + goto read_omap; + } + + try { + auto iter = map_bl.cbegin(); + decode(m, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: couldn't decode avail_pools" << dendl; + } + +read_omap: + if (m.empty()) { + ret = sysobj.omap().get_all(dpp, &m, y); + + write_map = true; + } + + if (ret < 0 || m.empty()) { + vector<rgw_pool> pools; + string s = string("default.") + default_storage_pool_suffix; + pools.push_back(rgw_pool(s)); + vector<int> retcodes; + bufferlist bl; + ret = rados_svc->pool().create(dpp, pools, &retcodes); + if (ret < 0) + return ret; + ret = sysobj.omap().set(dpp, s, bl, y); + if (ret < 0) + return ret; + m[s] = bl; + } + + if (write_map) { + bufferlist new_bl; + encode(m, new_bl); + ret = sysobj.wop().write(dpp, new_bl, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: could not save avail pools map info ret=" << ret << dendl; + } + } + + auto miter = m.begin(); + if (m.size() > 1) { + // choose a pool at random + auto r = ceph::util::generate_random_number<size_t>(0, m.size() - 1); + std::advance(miter, r); + } + pool_name = miter->first; + + rgw_pool pool = pool_name; + + rule_info->storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &pool, nullptr); + rule_info->data_extra_pool = pool_name; + rule_info->index_pool = pool_name; + rule_info->index_type = rgw::BucketIndexType::Normal; + + return 0; +} + +int RGWSI_Zone::update_placement_map(const DoutPrefixProvider *dpp, optional_yield y) +{ + bufferlist header; + map<string, bufferlist> m; + rgw_raw_obj obj(zone_params->domain_root, avail_pools); + + auto sysobj = sysobj_svc->get_obj(obj); + int ret = sysobj.omap().get_all(dpp, &m, y); + if (ret < 0) + return ret; + + bufferlist new_bl; + encode(m, new_bl); + ret = sysobj.wop().write(dpp, new_bl, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: could not save avail pools map info ret=" << ret << dendl; + } + + return ret; +} + +int RGWSI_Zone::add_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& new_pool, optional_yield y) +{ + int ret = rados_svc->pool(new_pool).lookup(); + if (ret < 0) { // DNE, or something + return ret; + } + + rgw_raw_obj obj(zone_params->domain_root, avail_pools); + auto sysobj = sysobj_svc->get_obj(obj); + + bufferlist empty_bl; + ret = sysobj.omap().set(dpp, new_pool.to_str(), empty_bl, y); + + // don't care about return value + update_placement_map(dpp, y); + + return ret; +} + +int RGWSI_Zone::remove_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& old_pool, optional_yield y) +{ + rgw_raw_obj obj(zone_params->domain_root, avail_pools); + auto sysobj = sysobj_svc->get_obj(obj); + int ret = sysobj.omap().del(dpp, old_pool.to_str(), y); + + // don't care about return value + update_placement_map(dpp, y); + + return ret; +} + +int RGWSI_Zone::list_placement_set(const DoutPrefixProvider *dpp, set<rgw_pool>& names, optional_yield y) +{ + bufferlist header; + map<string, bufferlist> m; + + rgw_raw_obj obj(zone_params->domain_root, avail_pools); + auto sysobj = sysobj_svc->get_obj(obj); + int ret = sysobj.omap().get_all(dpp, &m, y); + if (ret < 0) + return ret; + + names.clear(); + map<string, bufferlist>::iterator miter; + for (miter = m.begin(); miter != m.end(); ++miter) { + names.insert(rgw_pool(miter->first)); + } + + return names.size(); +} + +bool RGWSI_Zone::get_redirect_zone_endpoint(string *endpoint) +{ + if (zone_public_config->redirect_zone.empty()) { + return false; + } + + auto iter = zone_conn_map.find(zone_public_config->redirect_zone); + if (iter == zone_conn_map.end()) { + ldout(cct, 0) << "ERROR: cannot find entry for redirect zone: " << zone_public_config->redirect_zone << dendl; + return false; + } + + RGWRESTConn *conn = iter->second; + + int ret = conn->get_url(*endpoint); + if (ret < 0) { + ldout(cct, 0) << "ERROR: redirect zone, conn->get_endpoint() returned ret=" << ret << dendl; + return false; + } + + return true; +} + diff --git a/src/rgw/services/svc_zone.h b/src/rgw/services/svc_zone.h new file mode 100644 index 000000000..7b0a277c4 --- /dev/null +++ b/src/rgw/services/svc_zone.h @@ -0,0 +1,165 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" + + +class RGWSI_RADOS; +class RGWSI_SysObj; +class RGWSI_SyncModules; +class RGWSI_Bucket_Sync; + +class RGWRealm; +class RGWZoneGroup; +class RGWZone; +class RGWZoneParams; +class RGWPeriod; +class RGWZonePlacementInfo; + +class RGWBucketSyncPolicyHandler; + +class RGWRESTConn; + +struct rgw_sync_policy_info; + +class RGWSI_Zone : public RGWServiceInstance +{ + friend struct RGWServices_Def; + + RGWSI_SysObj *sysobj_svc{nullptr}; + RGWSI_RADOS *rados_svc{nullptr}; + RGWSI_SyncModules *sync_modules_svc{nullptr}; + RGWSI_Bucket_Sync *bucket_sync_svc{nullptr}; + + RGWRealm *realm{nullptr}; + RGWZoneGroup *zonegroup{nullptr}; + RGWZone *zone_public_config{nullptr}; /* external zone params, e.g., entrypoints, log flags, etc. */ + RGWZoneParams *zone_params{nullptr}; /* internal zone params, e.g., rados pools */ + RGWPeriod *current_period{nullptr}; + rgw_zone_id cur_zone_id; + uint32_t zone_short_id{0}; + bool writeable_zone{false}; + bool exports_data{false}; + + std::shared_ptr<RGWBucketSyncPolicyHandler> sync_policy_handler; + std::map<rgw_zone_id, std::shared_ptr<RGWBucketSyncPolicyHandler> > sync_policy_handlers; + + RGWRESTConn *rest_master_conn{nullptr}; + std::map<rgw_zone_id, RGWRESTConn *> zone_conn_map; + std::vector<const RGWZone*> data_sync_source_zones; + std::map<rgw_zone_id, RGWRESTConn *> zone_data_notify_to_map; + std::map<std::string, RGWRESTConn *> zonegroup_conn_map; + + std::map<std::string, rgw_zone_id> zone_id_by_name; + std::map<rgw_zone_id, RGWZone> zone_by_id; + + std::unique_ptr<rgw_sync_policy_info> sync_policy; + + void init(RGWSI_SysObj *_sysobj_svc, + RGWSI_RADOS *_rados_svc, + RGWSI_SyncModules *_sync_modules_svc, + RGWSI_Bucket_Sync *_bucket_sync_svc); + int do_start(optional_yield y, const DoutPrefixProvider *dpp) override; + void shutdown() override; + + int init_zg_from_period(const DoutPrefixProvider *dpp, optional_yield y); + int init_zg_from_local(const DoutPrefixProvider *dpp, optional_yield y); + + int update_placement_map(const DoutPrefixProvider *dpp, optional_yield y); + + int create_default_zg(const DoutPrefixProvider *dpp, optional_yield y); + int init_default_zone(const DoutPrefixProvider *dpp, optional_yield y); + + int search_realm_with_zone(const DoutPrefixProvider *dpp, + const rgw_zone_id& zid, + RGWRealm *prealm, + RGWPeriod *pperiod, + RGWZoneGroup *pzonegroup, + bool *pfound, + optional_yield y); +public: + RGWSI_Zone(CephContext *cct); + ~RGWSI_Zone(); + + const RGWZoneParams& get_zone_params() const; + const RGWPeriod& get_current_period() const; + const RGWRealm& get_realm() const; + const RGWZoneGroup& get_zonegroup() const; + int get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) const; + const RGWZone& get_zone() const; + + std::shared_ptr<RGWBucketSyncPolicyHandler> get_sync_policy_handler(std::optional<rgw_zone_id> zone = std::nullopt) const; + + const std::string& zone_name() const; + const rgw_zone_id& zone_id() const { + return cur_zone_id; + } + uint32_t get_zone_short_id() const; + + const std::string& get_current_period_id() const; + bool has_zonegroup_api(const std::string& api) const; + + bool zone_is_writeable(); + bool zone_syncs_from(const RGWZone& target_zone, const RGWZone& source_zone) const; + bool zone_syncs_from(const RGWZone& source_zone) const; + bool get_redirect_zone_endpoint(std::string *endpoint); + bool sync_module_supports_writes() const { return writeable_zone; } + bool sync_module_exports_data() const { return exports_data; } + + RGWRESTConn *get_master_conn() { + return rest_master_conn; + } + + std::map<std::string, RGWRESTConn *>& get_zonegroup_conn_map() { + return zonegroup_conn_map; + } + + std::map<rgw_zone_id, RGWRESTConn *>& get_zone_conn_map() { + return zone_conn_map; + } + + std::vector<const RGWZone*>& get_data_sync_source_zones() { + return data_sync_source_zones; + } + + std::map<rgw_zone_id, RGWRESTConn *>& get_zone_data_notify_to_map() { + return zone_data_notify_to_map; + } + + RGWZone* find_zone(const rgw_zone_id& id); + + RGWRESTConn *get_zone_conn(const rgw_zone_id& zone_id); + RGWRESTConn *get_zone_conn_by_name(const std::string& name); + bool find_zone_id_by_name(const std::string& name, rgw_zone_id *id); + + int select_bucket_placement(const DoutPrefixProvider *dpp, const RGWUserInfo& user_info, const std::string& zonegroup_id, + const rgw_placement_rule& rule, + rgw_placement_rule *pselected_rule, RGWZonePlacementInfo *rule_info, optional_yield y); + int select_legacy_bucket_placement(const DoutPrefixProvider *dpp, RGWZonePlacementInfo *rule_info, optional_yield y); + int select_new_bucket_location(const DoutPrefixProvider *dpp, const RGWUserInfo& user_info, const std::string& zonegroup_id, + const rgw_placement_rule& rule, + rgw_placement_rule *pselected_rule_name, RGWZonePlacementInfo *rule_info, + optional_yield y); + int select_bucket_location_by_rule(const DoutPrefixProvider *dpp, const rgw_placement_rule& location_rule, RGWZonePlacementInfo *rule_info, optional_yield y); + + int add_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& new_pool, optional_yield y); + int remove_bucket_placement(const DoutPrefixProvider *dpp, const rgw_pool& old_pool, optional_yield y); + int list_placement_set(const DoutPrefixProvider *dpp, std::set<rgw_pool>& names, optional_yield y); + + bool is_meta_master() const; + + bool need_to_sync() const; + bool need_to_log_data() const; + bool need_to_log_metadata() const; + bool can_reshard() const; + bool is_syncing_bucket_meta(const rgw_bucket& bucket); + + int list_zonegroups(const DoutPrefixProvider *dpp, std::list<std::string>& zonegroups); + int list_regions(const DoutPrefixProvider *dpp, std::list<std::string>& regions); + int list_zones(const DoutPrefixProvider *dpp, std::list<std::string>& zones); + int list_realms(const DoutPrefixProvider *dpp, std::list<std::string>& realms); + int list_periods(const DoutPrefixProvider *dpp, std::list<std::string>& periods); + int list_periods(const DoutPrefixProvider *dpp, const std::string& current_period, std::list<std::string>& periods, optional_yield y); +}; diff --git a/src/rgw/services/svc_zone_utils.cc b/src/rgw/services/svc_zone_utils.cc new file mode 100644 index 000000000..712bb97c9 --- /dev/null +++ b/src/rgw/services/svc_zone_utils.cc @@ -0,0 +1,64 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_zone_utils.h" +#include "svc_rados.h" +#include "svc_zone.h" + +#include "rgw_zone.h" + +using namespace std; + +int RGWSI_ZoneUtils::do_start(optional_yield, const DoutPrefixProvider *dpp) +{ + init_unique_trans_id_deps(); + + return 0; +} + +string RGWSI_ZoneUtils::gen_host_id() { + /* uint64_t needs 16, two '-' separators and a trailing null */ + const string& zone_name = zone_svc->get_zone().name; + const string& zonegroup_name = zone_svc->get_zonegroup().get_name(); + char charbuf[16 + zone_name.size() + zonegroup_name.size() + 2 + 1]; + snprintf(charbuf, sizeof(charbuf), "%llx-%s-%s", (unsigned long long)rados_svc->instance_id(), zone_name.c_str(), zonegroup_name.c_str()); + return string(charbuf); +} + +string RGWSI_ZoneUtils::unique_id(uint64_t unique_num) +{ + char buf[32]; + snprintf(buf, sizeof(buf), ".%llu.%llu", (unsigned long long)rados_svc->instance_id(), (unsigned long long)unique_num); + string s = zone_svc->get_zone_params().get_id() + buf; + return s; +} + +void RGWSI_ZoneUtils::init_unique_trans_id_deps() { + char buf[16 + 2 + 1]; /* uint64_t needs 16, 2 hyphens add further 2 */ + + snprintf(buf, sizeof(buf), "-%llx-", (unsigned long long)rados_svc->instance_id()); + url_encode(string(buf) + zone_svc->get_zone().name, trans_id_suffix); +} + +/* In order to preserve compatibility with Swift API, transaction ID + * should contain at least 32 characters satisfying following spec: + * - first 21 chars must be in range [0-9a-f]. Swift uses this + * space for storing fragment of UUID obtained through a call to + * uuid4() function of Python's uuid module; + * - char no. 22 must be a hyphen; + * - at least 10 next characters constitute hex-formatted timestamp + * padded with zeroes if necessary. All bytes must be in [0-9a-f] + * range; + * - last, optional part of transaction ID is any url-encoded string + * without restriction on length. */ +string RGWSI_ZoneUtils::unique_trans_id(const uint64_t unique_num) { + char buf[41]; /* 2 + 21 + 1 + 16 (timestamp can consume up to 16) + 1 */ + time_t timestamp = time(NULL); + + snprintf(buf, sizeof(buf), "tx%021llx-%010llx", + (unsigned long long)unique_num, + (unsigned long long)timestamp); + + return string(buf) + trans_id_suffix; +} + diff --git a/src/rgw/services/svc_zone_utils.h b/src/rgw/services/svc_zone_utils.h new file mode 100644 index 000000000..43e3fee8d --- /dev/null +++ b/src/rgw/services/svc_zone_utils.h @@ -0,0 +1,38 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include "rgw_service.h" + + +class RGWSI_RADOS; +class RGWSI_Zone; + +class RGWSI_ZoneUtils : public RGWServiceInstance +{ + friend struct RGWServices_Def; + + RGWSI_RADOS *rados_svc{nullptr}; + RGWSI_Zone *zone_svc{nullptr}; + + std::string trans_id_suffix; + + void init(RGWSI_RADOS *_rados_svc, + RGWSI_Zone *_zone_svc) { + rados_svc = _rados_svc; + zone_svc = _zone_svc; + } + + int do_start(optional_yield, const DoutPrefixProvider *dpp) override; + + void init_unique_trans_id_deps(); + +public: + RGWSI_ZoneUtils(CephContext *cct): RGWServiceInstance(cct) {} + + std::string gen_host_id(); + std::string unique_id(uint64_t unique_num); + + std::string unique_trans_id(const uint64_t unique_num); +}; |