summaryrefslogtreecommitdiffstats
path: root/src/kv
diff options
context:
space:
mode:
Diffstat (limited to 'src/kv')
-rw-r--r--src/kv/CMakeLists.txt13
-rw-r--r--src/kv/KeyValueDB.cc27
-rw-r--r--src/kv/KeyValueDB.h455
-rw-r--r--src/kv/KeyValueHistogram.cc78
-rw-r--r--src/kv/KeyValueHistogram.h38
-rw-r--r--src/kv/RocksDBStore.cc3488
-rw-r--r--src/kv/RocksDBStore.h551
-rw-r--r--src/kv/rocksdb_cache/BinnedLRUCache.cc719
-rw-r--r--src/kv/rocksdb_cache/BinnedLRUCache.h366
-rw-r--r--src/kv/rocksdb_cache/ShardedCache.cc182
-rw-r--r--src/kv/rocksdb_cache/ShardedCache.h197
11 files changed, 6114 insertions, 0 deletions
diff --git a/src/kv/CMakeLists.txt b/src/kv/CMakeLists.txt
new file mode 100644
index 000000000..9121a6257
--- /dev/null
+++ b/src/kv/CMakeLists.txt
@@ -0,0 +1,13 @@
+set(kv_srcs
+ KeyValueDB.cc
+ RocksDBStore.cc
+ KeyValueHistogram.cc
+ rocksdb_cache/ShardedCache.cc
+ rocksdb_cache/BinnedLRUCache.cc)
+
+add_library(kv STATIC ${kv_srcs}
+ $<TARGET_OBJECTS:common_prioritycache_obj>)
+
+target_link_libraries(kv
+ RocksDB::RocksDB
+ heap_profiler)
diff --git a/src/kv/KeyValueDB.cc b/src/kv/KeyValueDB.cc
new file mode 100644
index 000000000..83040023d
--- /dev/null
+++ b/src/kv/KeyValueDB.cc
@@ -0,0 +1,27 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "KeyValueDB.h"
+#include "RocksDBStore.h"
+
+using std::map;
+using std::string;
+
+KeyValueDB *KeyValueDB::create(CephContext *cct, const string& type,
+ const string& dir,
+ map<string,string> options,
+ void *p)
+{
+ if (type == "rocksdb") {
+ return new RocksDBStore(cct, dir, options, p);
+ }
+ return NULL;
+}
+
+int KeyValueDB::test_init(const string& type, const string& dir)
+{
+ if (type == "rocksdb") {
+ return RocksDBStore::_test_init(dir);
+ }
+ return -EINVAL;
+}
diff --git a/src/kv/KeyValueDB.h b/src/kv/KeyValueDB.h
new file mode 100644
index 000000000..9cfb44827
--- /dev/null
+++ b/src/kv/KeyValueDB.h
@@ -0,0 +1,455 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef KEY_VALUE_DB_H
+#define KEY_VALUE_DB_H
+
+#include "include/buffer.h"
+#include <ostream>
+#include <set>
+#include <map>
+#include <optional>
+#include <string>
+#include <boost/scoped_ptr.hpp>
+#include "include/encoding.h"
+#include "common/Formatter.h"
+#include "common/perf_counters.h"
+#include "common/PriorityCache.h"
+
+/**
+ * Defines virtual interface to be implemented by key value store
+ *
+ * Kyoto Cabinet should implement this
+ */
+class KeyValueDB {
+public:
+ class TransactionImpl {
+ public:
+ /// Set Keys
+ void set(
+ const std::string &prefix, ///< [in] Prefix for keys, or CF name
+ const std::map<std::string, ceph::buffer::list> &to_set ///< [in] keys/values to set
+ ) {
+ for (auto it = to_set.cbegin(); it != to_set.cend(); ++it)
+ set(prefix, it->first, it->second);
+ }
+
+ /// Set Keys (via encoded ceph::buffer::list)
+ void set(
+ const std::string &prefix, ///< [in] prefix, or CF name
+ ceph::buffer::list& to_set_bl ///< [in] encoded key/values to set
+ ) {
+ using ceph::decode;
+ auto p = std::cbegin(to_set_bl);
+ uint32_t num;
+ decode(num, p);
+ while (num--) {
+ std::string key;
+ ceph::buffer::list value;
+ decode(key, p);
+ decode(value, p);
+ set(prefix, key, value);
+ }
+ }
+
+ /// Set Key
+ virtual void set(
+ const std::string &prefix, ///< [in] Prefix or CF for the key
+ const std::string &k, ///< [in] Key to set
+ const ceph::buffer::list &bl ///< [in] Value to set
+ ) = 0;
+ virtual void set(
+ const std::string &prefix,
+ const char *k,
+ size_t keylen,
+ const ceph::buffer::list& bl) {
+ set(prefix, std::string(k, keylen), bl);
+ }
+
+ /// Removes Keys (via encoded ceph::buffer::list)
+ void rmkeys(
+ const std::string &prefix, ///< [in] Prefix or CF to search for
+ ceph::buffer::list &keys_bl ///< [in] Keys to remove
+ ) {
+ using ceph::decode;
+ auto p = std::cbegin(keys_bl);
+ uint32_t num;
+ decode(num, p);
+ while (num--) {
+ std::string key;
+ decode(key, p);
+ rmkey(prefix, key);
+ }
+ }
+
+ /// Removes Keys
+ void rmkeys(
+ const std::string &prefix, ///< [in] Prefix/CF to search for
+ const std::set<std::string> &keys ///< [in] Keys to remove
+ ) {
+ for (auto it = keys.cbegin(); it != keys.cend(); ++it)
+ rmkey(prefix, *it);
+ }
+
+ /// Remove Key
+ virtual void rmkey(
+ const std::string &prefix, ///< [in] Prefix/CF to search for
+ const std::string &k ///< [in] Key to remove
+ ) = 0;
+ virtual void rmkey(
+ const std::string &prefix, ///< [in] Prefix to search for
+ const char *k, ///< [in] Key to remove
+ size_t keylen
+ ) {
+ rmkey(prefix, std::string(k, keylen));
+ }
+
+ /// Remove Single Key which exists and was not overwritten.
+ /// This API is only related to performance optimization, and should only be
+ /// re-implemented by log-insert-merge tree based keyvalue stores(such as RocksDB).
+ /// If a key is overwritten (by calling set multiple times), then the result
+ /// of calling rm_single_key on this key is undefined.
+ virtual void rm_single_key(
+ const std::string &prefix, ///< [in] Prefix/CF to search for
+ const std::string &k ///< [in] Key to remove
+ ) { return rmkey(prefix, k);}
+
+ /// Removes keys beginning with prefix
+ virtual void rmkeys_by_prefix(
+ const std::string &prefix ///< [in] Prefix/CF by which to remove keys
+ ) = 0;
+
+ virtual void rm_range_keys(
+ const std::string &prefix, ///< [in] Prefix by which to remove keys
+ const std::string &start, ///< [in] The start bound of remove keys
+ const std::string &end ///< [in] The start bound of remove keys
+ ) = 0;
+
+ /// Merge value into key
+ virtual void merge(
+ const std::string &prefix, ///< [in] Prefix/CF ==> MUST match some established merge operator
+ const std::string &key, ///< [in] Key to be merged
+ const ceph::buffer::list &value ///< [in] value to be merged into key
+ ) { ceph_abort_msg("Not implemented"); }
+
+ virtual ~TransactionImpl() {}
+ };
+ typedef std::shared_ptr< TransactionImpl > Transaction;
+
+ /// create a new instance
+ static KeyValueDB *create(CephContext *cct, const std::string& type,
+ const std::string& dir,
+ std::map<std::string,std::string> options = {},
+ void *p = NULL);
+
+ /// test whether we can successfully initialize; may have side effects (e.g., create)
+ static int test_init(const std::string& type, const std::string& dir);
+ virtual int init(std::string option_str="") = 0;
+ virtual int open(std::ostream &out, const std::string& cfs="") = 0;
+ // std::vector cfs contains column families to be created when db is created.
+ virtual int create_and_open(std::ostream &out, const std::string& cfs="") = 0;
+
+ virtual int open_read_only(std::ostream &out, const std::string& cfs="") {
+ return -ENOTSUP;
+ }
+
+ virtual void close() { }
+
+ /// Try to repair K/V database. rocksdb requires that database must be not opened.
+ virtual int repair(std::ostream &out) { return 0; }
+
+ virtual Transaction get_transaction() = 0;
+ virtual int submit_transaction(Transaction) = 0;
+ virtual int submit_transaction_sync(Transaction t) {
+ return submit_transaction(t);
+ }
+
+ /// Retrieve Keys
+ virtual int get(
+ const std::string &prefix, ///< [in] Prefix/CF for key
+ const std::set<std::string> &key, ///< [in] Key to retrieve
+ std::map<std::string, ceph::buffer::list> *out ///< [out] Key value retrieved
+ ) = 0;
+ virtual int get(const std::string &prefix, ///< [in] prefix or CF name
+ const std::string &key, ///< [in] key
+ ceph::buffer::list *value) { ///< [out] value
+ std::set<std::string> ks;
+ ks.insert(key);
+ std::map<std::string,ceph::buffer::list> om;
+ int r = get(prefix, ks, &om);
+ if (om.find(key) != om.end()) {
+ *value = std::move(om[key]);
+ } else {
+ *value = ceph::buffer::list();
+ r = -ENOENT;
+ }
+ return r;
+ }
+ virtual int get(const std::string &prefix,
+ const char *key, size_t keylen,
+ ceph::buffer::list *value) {
+ return get(prefix, std::string(key, keylen), value);
+ }
+
+ // This superclass is used both by kv iterators *and* by the ObjectMap
+ // omap iterator. The class hierarchies are unfortunately tied together
+ // by the legacy DBOjectMap implementation :(.
+ class SimplestIteratorImpl {
+ public:
+ virtual int seek_to_first() = 0;
+ virtual int upper_bound(const std::string &after) = 0;
+ virtual int lower_bound(const std::string &to) = 0;
+ virtual bool valid() = 0;
+ virtual int next() = 0;
+ virtual std::string key() = 0;
+ virtual std::string tail_key() {
+ return "";
+ }
+ virtual ceph::buffer::list value() = 0;
+ virtual int status() = 0;
+ virtual ~SimplestIteratorImpl() {}
+ };
+
+ class IteratorImpl : public SimplestIteratorImpl {
+ public:
+ virtual ~IteratorImpl() {}
+ virtual int seek_to_last() = 0;
+ virtual int prev() = 0;
+ virtual std::pair<std::string, std::string> raw_key() = 0;
+ virtual ceph::buffer::ptr value_as_ptr() {
+ ceph::buffer::list bl = value();
+ if (bl.length() == 1) {
+ return *bl.buffers().begin();
+ } else if (bl.length() == 0) {
+ return ceph::buffer::ptr();
+ } else {
+ ceph_abort();
+ }
+ }
+ };
+ typedef std::shared_ptr< IteratorImpl > Iterator;
+
+ // This is the low-level iterator implemented by the underlying KV store.
+ class WholeSpaceIteratorImpl {
+ public:
+ virtual int seek_to_first() = 0;
+ virtual int seek_to_first(const std::string &prefix) = 0;
+ virtual int seek_to_last() = 0;
+ virtual int seek_to_last(const std::string &prefix) = 0;
+ virtual int upper_bound(const std::string &prefix, const std::string &after) = 0;
+ virtual int lower_bound(const std::string &prefix, const std::string &to) = 0;
+ virtual bool valid() = 0;
+ virtual int next() = 0;
+ virtual int prev() = 0;
+ virtual std::string key() = 0;
+ virtual std::pair<std::string,std::string> raw_key() = 0;
+ virtual bool raw_key_is_prefixed(const std::string &prefix) = 0;
+ virtual ceph::buffer::list value() = 0;
+ virtual ceph::buffer::ptr value_as_ptr() {
+ ceph::buffer::list bl = value();
+ if (bl.length()) {
+ return *bl.buffers().begin();
+ } else {
+ return ceph::buffer::ptr();
+ }
+ }
+ virtual int status() = 0;
+ virtual size_t key_size() {
+ return 0;
+ }
+ virtual size_t value_size() {
+ return 0;
+ }
+ virtual ~WholeSpaceIteratorImpl() { }
+ };
+ typedef std::shared_ptr< WholeSpaceIteratorImpl > WholeSpaceIterator;
+
+private:
+ // This class filters a WholeSpaceIterator by a prefix.
+ // Performs as a dummy wrapper over WholeSpaceIterator
+ // if prefix is empty
+ class PrefixIteratorImpl : public IteratorImpl {
+ const std::string prefix;
+ WholeSpaceIterator generic_iter;
+ public:
+ PrefixIteratorImpl(const std::string &prefix, WholeSpaceIterator iter) :
+ prefix(prefix), generic_iter(iter) { }
+ ~PrefixIteratorImpl() override { }
+
+ int seek_to_first() override {
+ return prefix.empty() ?
+ generic_iter->seek_to_first() :
+ generic_iter->seek_to_first(prefix);
+ }
+ int seek_to_last() override {
+ return prefix.empty() ?
+ generic_iter->seek_to_last() :
+ generic_iter->seek_to_last(prefix);
+ }
+ int upper_bound(const std::string &after) override {
+ return generic_iter->upper_bound(prefix, after);
+ }
+ int lower_bound(const std::string &to) override {
+ return generic_iter->lower_bound(prefix, to);
+ }
+ bool valid() override {
+ if (!generic_iter->valid())
+ return false;
+ if (prefix.empty())
+ return true;
+ return prefix.empty() ?
+ true :
+ generic_iter->raw_key_is_prefixed(prefix);
+ }
+ int next() override {
+ return generic_iter->next();
+ }
+ int prev() override {
+ return generic_iter->prev();
+ }
+ std::string key() override {
+ return generic_iter->key();
+ }
+ std::pair<std::string, std::string> raw_key() override {
+ return generic_iter->raw_key();
+ }
+ ceph::buffer::list value() override {
+ return generic_iter->value();
+ }
+ ceph::buffer::ptr value_as_ptr() override {
+ return generic_iter->value_as_ptr();
+ }
+ int status() override {
+ return generic_iter->status();
+ }
+ };
+protected:
+ Iterator make_iterator(const std::string &prefix, WholeSpaceIterator w_iter) {
+ return std::make_shared<PrefixIteratorImpl>(
+ prefix,
+ w_iter);
+ }
+public:
+ typedef uint32_t IteratorOpts;
+ static const uint32_t ITERATOR_NOCACHE = 1;
+
+ struct IteratorBounds {
+ std::optional<std::string> lower_bound;
+ std::optional<std::string> upper_bound;
+ };
+
+ virtual WholeSpaceIterator get_wholespace_iterator(IteratorOpts opts = 0) = 0;
+ virtual Iterator get_iterator(const std::string &prefix, IteratorOpts opts = 0, IteratorBounds bounds = IteratorBounds()) {
+ return make_iterator(prefix,
+ get_wholespace_iterator(opts));
+ }
+
+ virtual uint64_t get_estimated_size(std::map<std::string,uint64_t> &extra) = 0;
+ virtual int get_statfs(struct store_statfs_t *buf) {
+ return -EOPNOTSUPP;
+ }
+
+ virtual int set_cache_size(uint64_t) {
+ return -EOPNOTSUPP;
+ }
+
+ virtual int set_cache_high_pri_pool_ratio(double ratio) {
+ return -EOPNOTSUPP;
+ }
+
+ virtual int64_t get_cache_usage() const {
+ return -EOPNOTSUPP;
+ }
+
+ virtual int64_t get_cache_usage(std::string prefix) const {
+ return -EOPNOTSUPP;
+ }
+
+ virtual std::shared_ptr<PriorityCache::PriCache> get_priority_cache() const {
+ return nullptr;
+ }
+
+ virtual std::shared_ptr<PriorityCache::PriCache> get_priority_cache(std::string prefix) const {
+ return nullptr;
+ }
+
+
+
+ virtual ~KeyValueDB() {}
+
+ /// estimate space utilization for a prefix (in bytes)
+ virtual int64_t estimate_prefix_size(const std::string& prefix,
+ const std::string& key_prefix) {
+ return 0;
+ }
+
+ /// compact the underlying store
+ virtual void compact() {}
+
+ /// compact the underlying store in async mode
+ virtual void compact_async() {}
+
+ /// compact db for all keys with a given prefix
+ virtual void compact_prefix(const std::string& prefix) {}
+ /// compact db for all keys with a given prefix, async
+ virtual void compact_prefix_async(const std::string& prefix) {}
+ virtual void compact_range(const std::string& prefix,
+ const std::string& start, const std::string& end) {}
+ virtual void compact_range_async(const std::string& prefix,
+ const std::string& start, const std::string& end) {}
+
+ // See RocksDB merge operator definition, we support the basic
+ // associative merge only right now.
+ class MergeOperator {
+ public:
+ /// Merge into a key that doesn't exist
+ virtual void merge_nonexistent(
+ const char *rdata, size_t rlen,
+ std::string *new_value) = 0;
+ /// Merge into a key that does exist
+ virtual void merge(
+ const char *ldata, size_t llen,
+ const char *rdata, size_t rlen,
+ std::string *new_value) = 0;
+ /// We use each operator name and each prefix to construct the overall RocksDB operator name for consistency check at open time.
+ virtual const char *name() const = 0;
+
+ virtual ~MergeOperator() {}
+ };
+
+ /// Setup one or more operators, this needs to be done BEFORE the DB is opened.
+ virtual int set_merge_operator(const std::string& prefix,
+ std::shared_ptr<MergeOperator> mop) {
+ return -EOPNOTSUPP;
+ }
+
+ virtual void get_statistics(ceph::Formatter *f) {
+ return;
+ }
+
+ /**
+ * Return your perf counters if you have any. Subclasses are not
+ * required to implement this, and callers must respect a null return
+ * value.
+ */
+ virtual PerfCounters *get_perf_counters() {
+ return nullptr;
+ }
+
+ /**
+ * Access implementation specific integral property corresponding
+ * to passed property and prefic.
+ * Return value is true if property is valid for prefix, populates out.
+ */
+ virtual bool get_property(
+ const std::string &property,
+ uint64_t *out) {
+ return false;
+ }
+protected:
+ /// List of matching prefixes/ColumnFamilies and merge operators
+ std::vector<std::pair<std::string,
+ std::shared_ptr<MergeOperator> > > merge_ops;
+
+};
+
+#endif
diff --git a/src/kv/KeyValueHistogram.cc b/src/kv/KeyValueHistogram.cc
new file mode 100644
index 000000000..5015225d5
--- /dev/null
+++ b/src/kv/KeyValueHistogram.cc
@@ -0,0 +1,78 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/stringify.h"
+#include "KeyValueHistogram.h"
+using std::map;
+using std::string;
+using ceph::Formatter;
+
+#define KEY_SLAB 32
+#define VALUE_SLAB 64
+
+int KeyValueHistogram::get_key_slab(size_t sz)
+{
+ return (sz / KEY_SLAB);
+}
+
+string KeyValueHistogram::get_key_slab_to_range(int slab)
+{
+ int lower_bound = slab * KEY_SLAB;
+ int upper_bound = (slab + 1) * KEY_SLAB;
+ string ret = "[" + stringify(lower_bound) + "," + stringify(upper_bound) + ")";
+ return ret;
+}
+
+int KeyValueHistogram::get_value_slab(size_t sz)
+{
+ return (sz / VALUE_SLAB);
+}
+
+string KeyValueHistogram::get_value_slab_to_range(int slab)
+{
+ int lower_bound = slab * VALUE_SLAB;
+ int upper_bound = (slab + 1) * VALUE_SLAB;
+ string ret = "[" + stringify(lower_bound) + "," + stringify(upper_bound) + ")";
+ return ret;
+}
+
+void KeyValueHistogram::update_hist_entry(map<string, map<int, struct key_dist> >& key_hist,
+ const string& prefix, size_t key_size, size_t value_size)
+{
+ uint32_t key_slab = get_key_slab(key_size);
+ uint32_t value_slab = get_value_slab(value_size);
+ key_hist[prefix][key_slab].count++;
+ key_hist[prefix][key_slab].max_len =
+ std::max<size_t>(key_size, key_hist[prefix][key_slab].max_len);
+ key_hist[prefix][key_slab].val_map[value_slab].count++;
+ key_hist[prefix][key_slab].val_map[value_slab].max_len =
+ std::max<size_t>(value_size,
+ key_hist[prefix][key_slab].val_map[value_slab].max_len);
+}
+
+void KeyValueHistogram::dump(Formatter* f)
+{
+ f->open_object_section("rocksdb_value_distribution");
+ for (auto i : value_hist) {
+ f->dump_unsigned(get_value_slab_to_range(i.first).data(), i.second);
+ }
+ f->close_section();
+
+ f->open_object_section("rocksdb_key_value_histogram");
+ for (auto i : key_hist) {
+ f->dump_string("prefix", i.first);
+ f->open_object_section("key_hist");
+ for (auto k : i.second) {
+ f->dump_unsigned(get_key_slab_to_range(k.first).data(), k.second.count);
+ f->dump_unsigned("max_len", k.second.max_len);
+ f->open_object_section("value_hist");
+ for (auto j : k.second.val_map) {
+ f->dump_unsigned(get_value_slab_to_range(j.first).data(), j.second.count);
+ f->dump_unsigned("max_len", j.second.max_len);
+ }
+ f->close_section();
+ }
+ f->close_section();
+ }
+ f->close_section();
+}
diff --git a/src/kv/KeyValueHistogram.h b/src/kv/KeyValueHistogram.h
new file mode 100644
index 000000000..2da44bd4e
--- /dev/null
+++ b/src/kv/KeyValueHistogram.h
@@ -0,0 +1,38 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef KeyValueHistogram_H
+#define KeyValueHistogram_H
+
+#include <map>
+#include <string>
+#include "common/Formatter.h"
+
+/**
+ *
+ * Key Value DB Histogram generator
+ *
+ */
+struct KeyValueHistogram {
+ struct value_dist {
+ uint64_t count;
+ uint32_t max_len;
+ };
+
+ struct key_dist {
+ uint64_t count;
+ uint32_t max_len;
+ std::map<int, struct value_dist> val_map; ///< slab id to count, max length of value and key
+ };
+
+ std::map<std::string, std::map<int, struct key_dist> > key_hist;
+ std::map<int, uint64_t> value_hist;
+ int get_key_slab(size_t sz);
+ std::string get_key_slab_to_range(int slab);
+ int get_value_slab(size_t sz);
+ std::string get_value_slab_to_range(int slab);
+ void update_hist_entry(std::map<std::string, std::map<int, struct key_dist> >& key_hist,
+ const std::string& prefix, size_t key_size, size_t value_size);
+ void dump(ceph::Formatter* f);
+};
+
+#endif
diff --git a/src/kv/RocksDBStore.cc b/src/kv/RocksDBStore.cc
new file mode 100644
index 000000000..fd2b26f38
--- /dev/null
+++ b/src/kv/RocksDBStore.cc
@@ -0,0 +1,3488 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <filesystem>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "rocksdb/db.h"
+#include "rocksdb/table.h"
+#include "rocksdb/env.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/utilities/convenience.h"
+#include "rocksdb/utilities/table_properties_collectors.h"
+#include "rocksdb/merge_operator.h"
+
+#include "common/perf_counters.h"
+#include "common/PriorityCache.h"
+#include "include/common_fwd.h"
+#include "include/scope_guard.h"
+#include "include/str_list.h"
+#include "include/stringify.h"
+#include "include/str_map.h"
+#include "KeyValueDB.h"
+#include "RocksDBStore.h"
+
+#include "common/debug.h"
+
+#define dout_context cct
+#define dout_subsys ceph_subsys_rocksdb
+#undef dout_prefix
+#define dout_prefix *_dout << "rocksdb: "
+
+namespace fs = std::filesystem;
+
+using std::function;
+using std::list;
+using std::map;
+using std::ostream;
+using std::pair;
+using std::set;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using ceph::bufferlist;
+using ceph::bufferptr;
+using ceph::Formatter;
+
+static const char* sharding_def_dir = "sharding";
+static const char* sharding_def_file = "sharding/def";
+static const char* sharding_recreate = "sharding/recreate_columns";
+static const char* resharding_column_lock = "reshardingXcommencingXlocked";
+
+static bufferlist to_bufferlist(rocksdb::Slice in) {
+ bufferlist bl;
+ bl.append(bufferptr(in.data(), in.size()));
+ return bl;
+}
+
+static rocksdb::SliceParts prepare_sliceparts(const bufferlist &bl,
+ vector<rocksdb::Slice> *slices)
+{
+ unsigned n = 0;
+ for (auto& buf : bl.buffers()) {
+ (*slices)[n].data_ = buf.c_str();
+ (*slices)[n].size_ = buf.length();
+ n++;
+ }
+ return rocksdb::SliceParts(slices->data(), slices->size());
+}
+
+
+//
+// One of these for the default rocksdb column family, routing each prefix
+// to the appropriate MergeOperator.
+//
+class RocksDBStore::MergeOperatorRouter
+ : public rocksdb::AssociativeMergeOperator
+{
+ RocksDBStore& store;
+public:
+ const char *Name() const override {
+ // Construct a name that rocksDB will validate against. We want to
+ // do this in a way that doesn't constrain the ordering of calls
+ // to set_merge_operator, so sort the merge operators and then
+ // construct a name from all of those parts.
+ store.assoc_name.clear();
+ map<std::string,std::string> names;
+
+ for (auto& p : store.merge_ops) {
+ names[p.first] = p.second->name();
+ }
+ for (auto& p : names) {
+ store.assoc_name += '.';
+ store.assoc_name += p.first;
+ store.assoc_name += ':';
+ store.assoc_name += p.second;
+ }
+ return store.assoc_name.c_str();
+ }
+
+ explicit MergeOperatorRouter(RocksDBStore &_store) : store(_store) {}
+
+ bool Merge(const rocksdb::Slice& key,
+ const rocksdb::Slice* existing_value,
+ const rocksdb::Slice& value,
+ std::string* new_value,
+ rocksdb::Logger* logger) const override {
+ // for default column family
+ // extract prefix from key and compare against each registered merge op;
+ // even though merge operator for explicit CF is included in merge_ops,
+ // it won't be picked up, since it won't match.
+ for (auto& p : store.merge_ops) {
+ if (p.first.compare(0, p.first.length(),
+ key.data(), p.first.length()) == 0 &&
+ key.data()[p.first.length()] == 0) {
+ if (existing_value) {
+ p.second->merge(existing_value->data(), existing_value->size(),
+ value.data(), value.size(),
+ new_value);
+ } else {
+ p.second->merge_nonexistent(value.data(), value.size(), new_value);
+ }
+ break;
+ }
+ }
+ return true; // OK :)
+ }
+};
+
+//
+// One of these per non-default column family, linked directly to the
+// merge operator for that CF/prefix (if any).
+//
+class RocksDBStore::MergeOperatorLinker
+ : public rocksdb::AssociativeMergeOperator
+{
+private:
+ std::shared_ptr<KeyValueDB::MergeOperator> mop;
+public:
+ explicit MergeOperatorLinker(const std::shared_ptr<KeyValueDB::MergeOperator> &o) : mop(o) {}
+
+ const char *Name() const override {
+ return mop->name();
+ }
+
+ bool Merge(const rocksdb::Slice& key,
+ const rocksdb::Slice* existing_value,
+ const rocksdb::Slice& value,
+ std::string* new_value,
+ rocksdb::Logger* logger) const override {
+ if (existing_value) {
+ mop->merge(existing_value->data(), existing_value->size(),
+ value.data(), value.size(),
+ new_value);
+ } else {
+ mop->merge_nonexistent(value.data(), value.size(), new_value);
+ }
+ return true;
+ }
+};
+
+int RocksDBStore::set_merge_operator(
+ const string& prefix,
+ std::shared_ptr<KeyValueDB::MergeOperator> mop)
+{
+ // If you fail here, it's because you can't do this on an open database
+ ceph_assert(db == nullptr);
+ merge_ops.push_back(std::make_pair(prefix,mop));
+ return 0;
+}
+
+class CephRocksdbLogger : public rocksdb::Logger {
+ CephContext *cct;
+public:
+ explicit CephRocksdbLogger(CephContext *c) : cct(c) {
+ cct->get();
+ }
+ ~CephRocksdbLogger() override {
+ cct->put();
+ }
+
+ // Write an entry to the log file with the specified format.
+ void Logv(const char* format, va_list ap) override {
+ Logv(rocksdb::INFO_LEVEL, format, ap);
+ }
+
+ // Write an entry to the log file with the specified log level
+ // and format. Any log with level under the internal log level
+ // of *this (see @SetInfoLogLevel and @GetInfoLogLevel) will not be
+ // printed.
+ void Logv(const rocksdb::InfoLogLevel log_level, const char* format,
+ va_list ap) override {
+ int v = rocksdb::NUM_INFO_LOG_LEVELS - log_level - 1;
+ dout(ceph::dout::need_dynamic(v));
+ char buf[65536];
+ vsnprintf(buf, sizeof(buf), format, ap);
+ *_dout << buf << dendl;
+ }
+};
+
+rocksdb::Logger *create_rocksdb_ceph_logger()
+{
+ return new CephRocksdbLogger(g_ceph_context);
+}
+
+static int string2bool(const string &val, bool &b_val)
+{
+ if (strcasecmp(val.c_str(), "false") == 0) {
+ b_val = false;
+ return 0;
+ } else if (strcasecmp(val.c_str(), "true") == 0) {
+ b_val = true;
+ return 0;
+ } else {
+ std::string err;
+ int b = strict_strtol(val.c_str(), 10, &err);
+ if (!err.empty())
+ return -EINVAL;
+ b_val = !!b;
+ return 0;
+ }
+}
+
+namespace rocksdb {
+extern std::string trim(const std::string& str);
+}
+
+// this function is a modification of rocksdb's StringToMap:
+// 1) accepts ' \n ; as separators
+// 2) leaves compound options with enclosing { and }
+rocksdb::Status StringToMap(const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map)
+{
+ using rocksdb::Status;
+ using rocksdb::trim;
+ assert(opts_map);
+ // Example:
+ // opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
+ // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
+ size_t pos = 0;
+ std::string opts = trim(opts_str);
+ while (pos < opts.size()) {
+ size_t eq_pos = opts.find('=', pos);
+ if (eq_pos == std::string::npos) {
+ return Status::InvalidArgument("Mismatched key value pair, '=' expected");
+ }
+ std::string key = trim(opts.substr(pos, eq_pos - pos));
+ if (key.empty()) {
+ return Status::InvalidArgument("Empty key found");
+ }
+
+ // skip space after '=' and look for '{' for possible nested options
+ pos = eq_pos + 1;
+ while (pos < opts.size() && isspace(opts[pos])) {
+ ++pos;
+ }
+ // Empty value at the end
+ if (pos >= opts.size()) {
+ (*opts_map)[key] = "";
+ break;
+ }
+ if (opts[pos] == '{') {
+ int count = 1;
+ size_t brace_pos = pos + 1;
+ while (brace_pos < opts.size()) {
+ if (opts[brace_pos] == '{') {
+ ++count;
+ } else if (opts[brace_pos] == '}') {
+ --count;
+ if (count == 0) {
+ break;
+ }
+ }
+ ++brace_pos;
+ }
+ // found the matching closing brace
+ if (count == 0) {
+ //include both '{' and '}'
+ (*opts_map)[key] = trim(opts.substr(pos, brace_pos - pos + 1));
+ // skip all whitespace and move to the next ';,'
+ // brace_pos points to the matching '}'
+ pos = brace_pos + 1;
+ while (pos < opts.size() && isspace(opts[pos])) {
+ ++pos;
+ }
+ if (pos < opts.size() && opts[pos] != ';' && opts[pos] != ',') {
+ return Status::InvalidArgument(
+ "Unexpected chars after nested options");
+ }
+ ++pos;
+ } else {
+ return Status::InvalidArgument(
+ "Mismatched curly braces for nested options");
+ }
+ } else {
+ size_t sc_pos = opts.find_first_of(",;", pos);
+ if (sc_pos == std::string::npos) {
+ (*opts_map)[key] = trim(opts.substr(pos));
+ // It either ends with a trailing , ; or the last key-value pair
+ break;
+ } else {
+ (*opts_map)[key] = trim(opts.substr(pos, sc_pos - pos));
+ }
+ pos = sc_pos + 1;
+ }
+ }
+ return Status::OK();
+}
+
+int RocksDBStore::tryInterpret(const string &key, const string &val, rocksdb::Options &opt)
+{
+ if (key == "compaction_threads") {
+ std::string err;
+ int f = strict_iecstrtoll(val, &err);
+ if (!err.empty())
+ return -EINVAL;
+ //Low priority threadpool is used for compaction
+ opt.env->SetBackgroundThreads(f, rocksdb::Env::Priority::LOW);
+ } else if (key == "flusher_threads") {
+ std::string err;
+ int f = strict_iecstrtoll(val, &err);
+ if (!err.empty())
+ return -EINVAL;
+ //High priority threadpool is used for flusher
+ opt.env->SetBackgroundThreads(f, rocksdb::Env::Priority::HIGH);
+ } else if (key == "compact_on_mount") {
+ int ret = string2bool(val, compact_on_mount);
+ if (ret != 0)
+ return ret;
+ } else if (key == "disableWAL") {
+ int ret = string2bool(val, disableWAL);
+ if (ret != 0)
+ return ret;
+ } else {
+ //unrecognize config options.
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int RocksDBStore::ParseOptionsFromString(const string &opt_str, rocksdb::Options &opt)
+{
+ return ParseOptionsFromStringStatic(cct, opt_str, opt,
+ [&](const string& k, const string& v, rocksdb::Options& o) {
+ return tryInterpret(k, v, o);
+ }
+ );
+}
+
+int RocksDBStore::ParseOptionsFromStringStatic(
+ CephContext *cct,
+ const string& opt_str,
+ rocksdb::Options& opt,
+ function<int(const string&, const string&, rocksdb::Options&)> interp)
+{
+ // keep aligned with func tryInterpret
+ const set<string> need_interp_keys = {"compaction_threads", "flusher_threads", "compact_on_mount", "disableWAL"};
+ rocksdb::Status status;
+ std::unordered_map<std::string, std::string> str_map;
+ status = StringToMap(opt_str, &str_map);
+ if (!status.ok()) {
+ dout(5) << __func__ << " error '" << status.getState() <<
+ "' while parsing options '" << opt_str << "'" << dendl;
+ return -EINVAL;
+ }
+
+ for (auto it = str_map.begin(); it != str_map.end(); ++it) {
+ string this_opt = it->first + "=" + it->second;
+ rocksdb::Status status =
+ rocksdb::GetOptionsFromString(opt, this_opt, &opt);
+ int r = 0;
+ if (!status.ok()) {
+ if (interp != nullptr) {
+ r = interp(it->first, it->second, opt);
+ } else if (!need_interp_keys.count(it->first)) {
+ r = -1;
+ }
+ if (r < 0) {
+ derr << status.ToString() << dendl;
+ return -EINVAL;
+ }
+ }
+ lgeneric_dout(cct, 1) << " set rocksdb option " << it->first
+ << " = " << it->second << dendl;
+ }
+ return 0;
+}
+
+int RocksDBStore::init(string _options_str)
+{
+ options_str = _options_str;
+ rocksdb::Options opt;
+ //try parse options
+ if (options_str.length()) {
+ int r = ParseOptionsFromString(options_str, opt);
+ if (r != 0) {
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+
+int RocksDBStore::create_db_dir()
+{
+ if (env) {
+ unique_ptr<rocksdb::Directory> dir;
+ env->NewDirectory(path, &dir);
+ } else {
+ if (!fs::exists(path)) {
+ std::error_code ec;
+ if (!fs::create_directory(path, ec)) {
+ derr << __func__ << " failed to create " << path
+ << ": " << ec.message() << dendl;
+ return -ec.value();
+ }
+ fs::permissions(path,
+ fs::perms::owner_all |
+ fs::perms::group_read | fs::perms::group_exec |
+ fs::perms::others_read | fs::perms::others_exec);
+ }
+ }
+ return 0;
+}
+
+int RocksDBStore::install_cf_mergeop(
+ const string &key_prefix,
+ rocksdb::ColumnFamilyOptions *cf_opt)
+{
+ ceph_assert(cf_opt != nullptr);
+ cf_opt->merge_operator.reset();
+ for (auto& i : merge_ops) {
+ if (i.first == key_prefix) {
+ cf_opt->merge_operator.reset(new MergeOperatorLinker(i.second));
+ }
+ }
+ return 0;
+}
+
+int RocksDBStore::create_and_open(ostream &out,
+ const std::string& cfs)
+{
+ int r = create_db_dir();
+ if (r < 0)
+ return r;
+ return do_open(out, true, false, cfs);
+}
+
+std::shared_ptr<rocksdb::Cache> RocksDBStore::create_block_cache(
+ const std::string& cache_type, size_t cache_size, double cache_prio_high) {
+ std::shared_ptr<rocksdb::Cache> cache;
+ auto shard_bits = cct->_conf->rocksdb_cache_shard_bits;
+ if (cache_type == "binned_lru") {
+ cache = rocksdb_cache::NewBinnedLRUCache(cct, cache_size, shard_bits, false, cache_prio_high);
+ } else if (cache_type == "lru") {
+ cache = rocksdb::NewLRUCache(cache_size, shard_bits);
+ } else if (cache_type == "clock") {
+ cache = rocksdb::NewClockCache(cache_size, shard_bits);
+ if (!cache) {
+ derr << "rocksdb_cache_type '" << cache
+ << "' chosen, but RocksDB not compiled with LibTBB. "
+ << dendl;
+ }
+ } else {
+ derr << "unrecognized rocksdb_cache_type '" << cache_type << "'" << dendl;
+ }
+ return cache;
+}
+
+int RocksDBStore::load_rocksdb_options(bool create_if_missing, rocksdb::Options& opt)
+{
+ rocksdb::Status status;
+
+ if (options_str.length()) {
+ int r = ParseOptionsFromString(options_str, opt);
+ if (r != 0) {
+ return -EINVAL;
+ }
+ }
+
+ if (cct->_conf->rocksdb_perf) {
+ dbstats = rocksdb::CreateDBStatistics();
+ opt.statistics = dbstats;
+ }
+
+ opt.create_if_missing = create_if_missing;
+ if (kv_options.count("separate_wal_dir")) {
+ opt.wal_dir = path + ".wal";
+ }
+
+ // Since ceph::for_each_substr doesn't return a value and
+ // std::stoull does throw, we may as well just catch everything here.
+ try {
+ if (kv_options.count("db_paths")) {
+ list<string> paths;
+ get_str_list(kv_options["db_paths"], "; \t", paths);
+ for (auto& p : paths) {
+ size_t pos = p.find(',');
+ if (pos == std::string::npos) {
+ derr << __func__ << " invalid db path item " << p << " in "
+ << kv_options["db_paths"] << dendl;
+ return -EINVAL;
+ }
+ string path = p.substr(0, pos);
+ string size_str = p.substr(pos + 1);
+ uint64_t size = atoll(size_str.c_str());
+ if (!size) {
+ derr << __func__ << " invalid db path item " << p << " in "
+ << kv_options["db_paths"] << dendl;
+ return -EINVAL;
+ }
+ opt.db_paths.push_back(rocksdb::DbPath(path, size));
+ dout(10) << __func__ << " db_path " << path << " size " << size << dendl;
+ }
+ }
+ } catch (const std::system_error& e) {
+ return -e.code().value();
+ }
+
+ if (cct->_conf->rocksdb_log_to_ceph_log) {
+ opt.info_log.reset(new CephRocksdbLogger(cct));
+ }
+
+ if (priv) {
+ dout(10) << __func__ << " using custom Env " << priv << dendl;
+ opt.env = static_cast<rocksdb::Env*>(priv);
+ } else {
+ env = opt.env;
+ }
+
+ opt.env->SetAllowNonOwnerAccess(false);
+
+ // caches
+ if (!set_cache_flag) {
+ cache_size = cct->_conf->rocksdb_cache_size;
+ }
+ uint64_t row_cache_size = cache_size * cct->_conf->rocksdb_cache_row_ratio;
+ uint64_t block_cache_size = cache_size - row_cache_size;
+
+ bbt_opts.block_cache = create_block_cache(cct->_conf->rocksdb_cache_type, block_cache_size);
+ if (!bbt_opts.block_cache) {
+ return -EINVAL;
+ }
+ bbt_opts.block_size = cct->_conf->rocksdb_block_size;
+
+ if (row_cache_size > 0)
+ opt.row_cache = rocksdb::NewLRUCache(row_cache_size,
+ cct->_conf->rocksdb_cache_shard_bits);
+ uint64_t bloom_bits = cct->_conf.get_val<uint64_t>("rocksdb_bloom_bits_per_key");
+ if (bloom_bits > 0) {
+ dout(10) << __func__ << " set bloom filter bits per key to "
+ << bloom_bits << dendl;
+ bbt_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(bloom_bits));
+ }
+ using std::placeholders::_1;
+ if (cct->_conf.with_val<std::string>("rocksdb_index_type",
+ std::bind(std::equal_to<std::string>(), _1,
+ "binary_search")))
+ bbt_opts.index_type = rocksdb::BlockBasedTableOptions::IndexType::kBinarySearch;
+ if (cct->_conf.with_val<std::string>("rocksdb_index_type",
+ std::bind(std::equal_to<std::string>(), _1,
+ "hash_search")))
+ bbt_opts.index_type = rocksdb::BlockBasedTableOptions::IndexType::kHashSearch;
+ if (cct->_conf.with_val<std::string>("rocksdb_index_type",
+ std::bind(std::equal_to<std::string>(), _1,
+ "two_level")))
+ bbt_opts.index_type = rocksdb::BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+ if (!bbt_opts.no_block_cache) {
+ bbt_opts.cache_index_and_filter_blocks =
+ cct->_conf.get_val<bool>("rocksdb_cache_index_and_filter_blocks");
+ bbt_opts.cache_index_and_filter_blocks_with_high_priority =
+ cct->_conf.get_val<bool>("rocksdb_cache_index_and_filter_blocks_with_high_priority");
+ bbt_opts.pin_l0_filter_and_index_blocks_in_cache =
+ cct->_conf.get_val<bool>("rocksdb_pin_l0_filter_and_index_blocks_in_cache");
+ }
+ bbt_opts.partition_filters = cct->_conf.get_val<bool>("rocksdb_partition_filters");
+ if (cct->_conf.get_val<Option::size_t>("rocksdb_metadata_block_size") > 0)
+ bbt_opts.metadata_block_size = cct->_conf.get_val<Option::size_t>("rocksdb_metadata_block_size");
+
+ opt.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbt_opts));
+ dout(10) << __func__ << " block size " << cct->_conf->rocksdb_block_size
+ << ", block_cache size " << byte_u_t(block_cache_size)
+ << ", row_cache size " << byte_u_t(row_cache_size)
+ << "; shards "
+ << (1 << cct->_conf->rocksdb_cache_shard_bits)
+ << ", type " << cct->_conf->rocksdb_cache_type
+ << dendl;
+
+ opt.merge_operator.reset(new MergeOperatorRouter(*this));
+ comparator = opt.comparator;
+ return 0;
+}
+
+void RocksDBStore::add_column_family(const std::string& cf_name, uint32_t hash_l, uint32_t hash_h,
+ size_t shard_idx, rocksdb::ColumnFamilyHandle *handle) {
+ dout(10) << __func__ << " column_name=" << cf_name << " shard_idx=" << shard_idx <<
+ " hash_l=" << hash_l << " hash_h=" << hash_h << " handle=" << (void*) handle << dendl;
+ bool exists = cf_handles.count(cf_name) > 0;
+ auto& column = cf_handles[cf_name];
+ if (exists) {
+ ceph_assert(hash_l == column.hash_l);
+ ceph_assert(hash_h == column.hash_h);
+ } else {
+ ceph_assert(hash_l < hash_h);
+ column.hash_l = hash_l;
+ column.hash_h = hash_h;
+ }
+ if (column.handles.size() <= shard_idx)
+ column.handles.resize(shard_idx + 1);
+ column.handles[shard_idx] = handle;
+ cf_ids_to_prefix.emplace(handle->GetID(), cf_name);
+}
+
+bool RocksDBStore::is_column_family(const std::string& prefix) {
+ return cf_handles.count(prefix);
+}
+
+std::string_view RocksDBStore::get_key_hash_view(const prefix_shards& shards, const char* key, const size_t keylen) {
+ uint32_t hash_l = std::min<uint32_t>(shards.hash_l, keylen);
+ uint32_t hash_h = std::min<uint32_t>(shards.hash_h, keylen);
+ return { key + hash_l, hash_h - hash_l };
+}
+
+rocksdb::ColumnFamilyHandle *RocksDBStore::get_key_cf(const prefix_shards& shards, const char* key, const size_t keylen) {
+ auto sv = get_key_hash_view(shards, key, keylen);
+ uint32_t hash = ceph_str_hash_rjenkins(sv.data(), sv.size());
+ return shards.handles[hash % shards.handles.size()];
+}
+
+rocksdb::ColumnFamilyHandle *RocksDBStore::get_cf_handle(const std::string& prefix, const std::string& key) {
+ auto iter = cf_handles.find(prefix);
+ if (iter == cf_handles.end()) {
+ return nullptr;
+ } else {
+ if (iter->second.handles.size() == 1) {
+ return iter->second.handles[0];
+ } else {
+ return get_key_cf(iter->second, key.data(), key.size());
+ }
+ }
+}
+
+rocksdb::ColumnFamilyHandle *RocksDBStore::get_cf_handle(const std::string& prefix, const char* key, size_t keylen) {
+ auto iter = cf_handles.find(prefix);
+ if (iter == cf_handles.end()) {
+ return nullptr;
+ } else {
+ if (iter->second.handles.size() == 1) {
+ return iter->second.handles[0];
+ } else {
+ return get_key_cf(iter->second, key, keylen);
+ }
+ }
+}
+
+/**
+ * If the specified IteratorBounds arg has both an upper and a lower bound defined, and they have equal placement hash
+ * strings, we can be sure that the entire iteration range exists in a single CF. In that case, we return the relevant
+ * CF handle. In all other cases, we return a nullptr to indicate that the specified bounds cannot necessarily be mapped
+ * to a single CF.
+ */
+rocksdb::ColumnFamilyHandle *RocksDBStore::check_cf_handle_bounds(const cf_handles_iterator& iter, const IteratorBounds& bounds) {
+ if (!bounds.lower_bound || !bounds.upper_bound) {
+ return nullptr;
+ }
+ ceph_assert(iter != cf_handles.end());
+ ceph_assert(iter->second.handles.size() != 1);
+ if (iter->second.hash_l != 0) {
+ return nullptr;
+ }
+ auto lower_bound_hash_str = get_key_hash_view(iter->second, bounds.lower_bound->data(), bounds.lower_bound->size());
+ auto upper_bound_hash_str = get_key_hash_view(iter->second, bounds.upper_bound->data(), bounds.upper_bound->size());
+ if (lower_bound_hash_str == upper_bound_hash_str) {
+ auto key = *bounds.lower_bound;
+ return get_key_cf(iter->second, key.data(), key.size());
+ } else {
+ return nullptr;
+ }
+}
+
+/**
+ * Definition of sharding:
+ * space-separated list of: column_def [ '=' options ]
+ * column_def := column_name '(' shard_count ')'
+ * column_def := column_name '(' shard_count ',' hash_begin '-' ')'
+ * column_def := column_name '(' shard_count ',' hash_begin '-' hash_end ')'
+ * I=write_buffer_size=1048576 O(6) m(7,10-) prefix(4,0-10)=disable_auto_compactions=true,max_bytes_for_level_base=1048576
+ */
+bool RocksDBStore::parse_sharding_def(const std::string_view text_def_in,
+ std::vector<ColumnFamily>& sharding_def,
+ char const* *error_position,
+ std::string *error_msg)
+{
+ std::string_view text_def = text_def_in;
+ char const* error_position_local = nullptr;
+ std::string error_msg_local;
+ if (error_position == nullptr) {
+ error_position = &error_position_local;
+ }
+ *error_position = nullptr;
+ if (error_msg == nullptr) {
+ error_msg = &error_msg_local;
+ error_msg->clear();
+ }
+
+ sharding_def.clear();
+ while (!text_def.empty()) {
+ std::string_view options;
+ std::string_view name;
+ size_t shard_cnt = 1;
+ uint32_t l_bound = 0;
+ uint32_t h_bound = std::numeric_limits<uint32_t>::max();
+
+ std::string_view column_def;
+ size_t spos = text_def.find(' ');
+ if (spos == std::string_view::npos) {
+ column_def = text_def;
+ text_def = std::string_view(text_def.end(), 0);
+ } else {
+ column_def = text_def.substr(0, spos);
+ text_def = text_def.substr(spos + 1);
+ }
+ size_t eqpos = column_def.find('=');
+ if (eqpos != std::string_view::npos) {
+ options = column_def.substr(eqpos + 1);
+ column_def = column_def.substr(0, eqpos);
+ }
+
+ size_t bpos = column_def.find('(');
+ if (bpos != std::string_view::npos) {
+ name = column_def.substr(0, bpos);
+ const char* nptr = &column_def[bpos + 1];
+ char* endptr;
+ shard_cnt = strtol(nptr, &endptr, 10);
+ if (nptr == endptr) {
+ *error_position = nptr;
+ *error_msg = "expecting integer";
+ break;
+ }
+ nptr = endptr;
+ if (*nptr == ',') {
+ nptr++;
+ l_bound = strtol(nptr, &endptr, 10);
+ if (nptr == endptr) {
+ *error_position = nptr;
+ *error_msg = "expecting integer";
+ break;
+ }
+ nptr = endptr;
+ if (*nptr != '-') {
+ *error_position = nptr;
+ *error_msg = "expecting '-'";
+ break;
+ }
+ nptr++;
+ h_bound = strtol(nptr, &endptr, 10);
+ if (nptr == endptr) {
+ h_bound = std::numeric_limits<uint32_t>::max();
+ }
+ nptr = endptr;
+ }
+ if (*nptr != ')') {
+ *error_position = nptr;
+ *error_msg = "expecting ')'";
+ break;
+ }
+ } else {
+ name = column_def;
+ }
+ sharding_def.emplace_back(std::string(name), shard_cnt, std::string(options), l_bound, h_bound);
+ }
+ return *error_position == nullptr;
+}
+
+void RocksDBStore::sharding_def_to_columns(const std::vector<ColumnFamily>& sharding_def,
+ std::vector<std::string>& columns)
+{
+ columns.clear();
+ for (size_t i = 0; i < sharding_def.size(); i++) {
+ if (sharding_def[i].shard_cnt == 1) {
+ columns.push_back(sharding_def[i].name);
+ } else {
+ for (size_t j = 0; j < sharding_def[i].shard_cnt; j++) {
+ columns.push_back(sharding_def[i].name + "-" + std::to_string(j));
+ }
+ }
+ }
+}
+
+int RocksDBStore::create_shards(const rocksdb::Options& opt,
+ const std::vector<ColumnFamily>& sharding_def)
+{
+ for (auto& p : sharding_def) {
+ // copy default CF settings, block cache, merge operators as
+ // the base for new CF
+ rocksdb::ColumnFamilyOptions cf_opt(opt);
+ rocksdb::Status status;
+ // apply options to column family
+ int r = update_column_family_options(p.name, p.options, &cf_opt);
+ if (r != 0) {
+ return r;
+ }
+ for (size_t idx = 0; idx < p.shard_cnt; idx++) {
+ std::string cf_name;
+ if (p.shard_cnt == 1)
+ cf_name = p.name;
+ else
+ cf_name = p.name + "-" + std::to_string(idx);
+ rocksdb::ColumnFamilyHandle *cf;
+ status = db->CreateColumnFamily(cf_opt, cf_name, &cf);
+ if (!status.ok()) {
+ derr << __func__ << " Failed to create rocksdb column family: "
+ << cf_name << dendl;
+ return -EINVAL;
+ }
+ // store the new CF handle
+ add_column_family(p.name, p.hash_l, p.hash_h, idx, cf);
+ }
+ }
+ return 0;
+}
+
+int RocksDBStore::apply_sharding(const rocksdb::Options& opt,
+ const std::string& sharding_text)
+{
+ // create and open column families
+ if (!sharding_text.empty()) {
+ bool b;
+ int r;
+ rocksdb::Status status;
+ std::vector<ColumnFamily> sharding_def;
+ char const* error_position;
+ std::string error_msg;
+ b = parse_sharding_def(sharding_text, sharding_def, &error_position, &error_msg);
+ if (!b) {
+ dout(1) << __func__ << " bad sharding: " << dendl;
+ dout(1) << __func__ << sharding_text << dendl;
+ dout(1) << __func__ << std::string(error_position - &sharding_text[0], ' ') << "^" << error_msg << dendl;
+ return -EINVAL;
+ }
+ r = create_shards(opt, sharding_def);
+ if (r != 0 ) {
+ derr << __func__ << " create_shards failed error=" << r << dendl;
+ return r;
+ }
+ opt.env->CreateDir(sharding_def_dir);
+ status = rocksdb::WriteStringToFile(opt.env, sharding_text,
+ sharding_def_file, true);
+ if (!status.ok()) {
+ derr << __func__ << " cannot write to " << sharding_def_file << dendl;
+ return -EIO;
+ }
+ } else {
+ opt.env->DeleteFile(sharding_def_file);
+ }
+ return 0;
+}
+
+// linking to rocksdb function defined in options_helper.cc
+// it can parse nested params like "nested_opt={opt1=1;opt2=2}"
+extern rocksdb::Status rocksdb::StringToMap(const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map);
+
+// Splits column family options from single string into name->value column_opts_map.
+// The split is done using RocksDB parser that understands "{" and "}", so it
+// properly extracts compound options.
+// If non-RocksDB option "block_cache" is defined it is extracted to block_cache_opt.
+int RocksDBStore::split_column_family_options(const std::string& options,
+ std::unordered_map<std::string, std::string>* opt_map,
+ std::string* block_cache_opt)
+{
+ dout(20) << __func__ << " options=" << options << dendl;
+ rocksdb::Status status = rocksdb::StringToMap(options, opt_map);
+ if (!status.ok()) {
+ dout(5) << __func__ << " error '" << status.getState()
+ << "' while parsing options '" << options << "'" << dendl;
+ return -EINVAL;
+ }
+ // if "block_cache" option exists, then move it to separate string
+ if (auto it = opt_map->find("block_cache"); it != opt_map->end()) {
+ *block_cache_opt = it->second;
+ opt_map->erase(it);
+ } else {
+ block_cache_opt->clear();
+ }
+ return 0;
+}
+
+// Updates column family options.
+// Take options from more_options and apply them to cf_opt.
+// Allowed options are exactly the same as allowed for column families in RocksDB.
+// Ceph addition is "block_cache" option that is translated to block_cache and
+// allows to specialize separate block cache for O column family.
+//
+// base_name - name of column without shard suffix: "-"+number
+// options - additional options to apply
+// cf_opt - column family options to update
+int RocksDBStore::update_column_family_options(const std::string& base_name,
+ const std::string& more_options,
+ rocksdb::ColumnFamilyOptions* cf_opt)
+{
+ std::unordered_map<std::string, std::string> options_map;
+ std::string block_cache_opt;
+ rocksdb::Status status;
+ int r = split_column_family_options(more_options, &options_map, &block_cache_opt);
+ if (r != 0) {
+ dout(5) << __func__ << " failed to parse options; column family=" << base_name
+ << " options=" << more_options << dendl;
+ return r;
+ }
+ status = rocksdb::GetColumnFamilyOptionsFromMap(*cf_opt, options_map, cf_opt);
+ if (!status.ok()) {
+ dout(5) << __func__ << " invalid column family optionsp; column family="
+ << base_name << " options=" << more_options << dendl;
+ dout(5) << __func__ << " RocksDB error='" << status.getState() << "'" << dendl;
+ return -EINVAL;
+ }
+ if (base_name != rocksdb::kDefaultColumnFamilyName) {
+ // default cf has its merge operator defined in load_rocksdb_options, should not override it
+ install_cf_mergeop(base_name, cf_opt);
+ }
+ if (!block_cache_opt.empty()) {
+ r = apply_block_cache_options(base_name, block_cache_opt, cf_opt);
+ if (r != 0) {
+ // apply_block_cache_options already does all necessary douts
+ return r;
+ }
+ }
+
+ // Set Compact on Deletion Factory
+ if (cct->_conf->rocksdb_cf_compact_on_deletion) {
+ size_t sliding_window = cct->_conf->rocksdb_cf_compact_on_deletion_sliding_window;
+ size_t trigger = cct->_conf->rocksdb_cf_compact_on_deletion_trigger;
+ cf_opt->table_properties_collector_factories.emplace_back(
+ rocksdb::NewCompactOnDeletionCollectorFactory(sliding_window, trigger));
+ }
+ return 0;
+}
+
+int RocksDBStore::apply_block_cache_options(const std::string& column_name,
+ const std::string& block_cache_opt,
+ rocksdb::ColumnFamilyOptions* cf_opt)
+{
+ rocksdb::Status status;
+ std::unordered_map<std::string, std::string> cache_options_map;
+ status = rocksdb::StringToMap(block_cache_opt, &cache_options_map);
+ if (!status.ok()) {
+ dout(5) << __func__ << " invalid block cache options; column=" << column_name
+ << " options=" << block_cache_opt << dendl;
+ dout(5) << __func__ << " RocksDB error='" << status.getState() << "'" << dendl;
+ return -EINVAL;
+ }
+ bool require_new_block_cache = false;
+ std::string cache_type = cct->_conf->rocksdb_cache_type;
+ if (const auto it = cache_options_map.find("type"); it != cache_options_map.end()) {
+ cache_type = it->second;
+ cache_options_map.erase(it);
+ require_new_block_cache = true;
+ }
+ size_t cache_size = cct->_conf->rocksdb_cache_size;
+ if (auto it = cache_options_map.find("size"); it != cache_options_map.end()) {
+ std::string error;
+ cache_size = strict_iecstrtoll(it->second.c_str(), &error);
+ if (!error.empty()) {
+ dout(10) << __func__ << " invalid size: '" << it->second << "'" << dendl;
+ return -EINVAL;
+ }
+ cache_options_map.erase(it);
+ require_new_block_cache = true;
+ }
+ double high_pri_pool_ratio = 0.0;
+ if (auto it = cache_options_map.find("high_ratio"); it != cache_options_map.end()) {
+ std::string error;
+ high_pri_pool_ratio = strict_strtod(it->second.c_str(), &error);
+ if (!error.empty()) {
+ dout(10) << __func__ << " invalid high_pri (float): '" << it->second << "'" << dendl;
+ return -EINVAL;
+ }
+ cache_options_map.erase(it);
+ require_new_block_cache = true;
+ }
+
+ rocksdb::BlockBasedTableOptions column_bbt_opts;
+ status = GetBlockBasedTableOptionsFromMap(bbt_opts, cache_options_map, &column_bbt_opts);
+ if (!status.ok()) {
+ dout(5) << __func__ << " invalid block cache options; column=" << column_name
+ << " options=" << block_cache_opt << dendl;
+ dout(5) << __func__ << " RocksDB error='" << status.getState() << "'" << dendl;
+ return -EINVAL;
+ }
+ std::shared_ptr<rocksdb::Cache> block_cache;
+ if (column_bbt_opts.no_block_cache) {
+ // clear all settings except no_block_cache
+ // rocksdb does not like then
+ column_bbt_opts = rocksdb::BlockBasedTableOptions();
+ column_bbt_opts.no_block_cache = true;
+ } else {
+ if (require_new_block_cache) {
+ block_cache = create_block_cache(cache_type, cache_size, high_pri_pool_ratio);
+ if (!block_cache) {
+ dout(5) << __func__ << " failed to create block cache for params: " << block_cache_opt << dendl;
+ return -EINVAL;
+ }
+ } else {
+ block_cache = bbt_opts.block_cache;
+ }
+ }
+ column_bbt_opts.block_cache = block_cache;
+ cf_bbt_opts[column_name] = column_bbt_opts;
+ cf_opt->table_factory.reset(NewBlockBasedTableFactory(cf_bbt_opts[column_name]));
+ return 0;
+}
+
+int RocksDBStore::verify_sharding(const rocksdb::Options& opt,
+ std::vector<rocksdb::ColumnFamilyDescriptor>& existing_cfs,
+ std::vector<std::pair<size_t, RocksDBStore::ColumnFamily> >& existing_cfs_shard,
+ std::vector<rocksdb::ColumnFamilyDescriptor>& missing_cfs,
+ std::vector<std::pair<size_t, RocksDBStore::ColumnFamily> >& missing_cfs_shard)
+{
+ rocksdb::Status status;
+ std::string stored_sharding_text;
+ status = opt.env->FileExists(sharding_def_file);
+ if (status.ok()) {
+ status = rocksdb::ReadFileToString(opt.env,
+ sharding_def_file,
+ &stored_sharding_text);
+ if(!status.ok()) {
+ derr << __func__ << " cannot read from " << sharding_def_file << dendl;
+ return -EIO;
+ }
+ dout(20) << __func__ << " sharding=" << stored_sharding_text << dendl;
+ } else {
+ dout(30) << __func__ << " no sharding" << dendl;
+ //no "sharding_def" present
+ }
+ //check if sharding_def matches stored_sharding_def
+ std::vector<ColumnFamily> stored_sharding_def;
+ parse_sharding_def(stored_sharding_text, stored_sharding_def);
+
+ std::sort(stored_sharding_def.begin(), stored_sharding_def.end(),
+ [](ColumnFamily& a, ColumnFamily& b) { return a.name < b.name; } );
+
+ std::vector<string> rocksdb_cfs;
+ status = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(opt),
+ path, &rocksdb_cfs);
+ if (!status.ok()) {
+ derr << __func__ << " unable to list column families: " << status.ToString() << dendl;
+ return -EIO;
+ }
+ dout(5) << __func__ << " column families from rocksdb: " << rocksdb_cfs << dendl;
+
+ auto emplace_cf = [&] (const RocksDBStore::ColumnFamily& column,
+ int32_t shard_id,
+ const std::string& shard_name,
+ const rocksdb::ColumnFamilyOptions& opt) {
+ if (std::find(rocksdb_cfs.begin(), rocksdb_cfs.end(), shard_name) != rocksdb_cfs.end()) {
+ existing_cfs.emplace_back(shard_name, opt);
+ existing_cfs_shard.emplace_back(shard_id, column);
+ } else {
+ missing_cfs.emplace_back(shard_name, opt);
+ missing_cfs_shard.emplace_back(shard_id, column);
+ }
+ };
+
+ for (auto& column : stored_sharding_def) {
+ rocksdb::ColumnFamilyOptions cf_opt(opt);
+ int r = update_column_family_options(column.name, column.options, &cf_opt);
+ if (r != 0) {
+ return r;
+ }
+ if (column.shard_cnt == 1) {
+ emplace_cf(column, 0, column.name, cf_opt);
+ } else {
+ for (size_t i = 0; i < column.shard_cnt; i++) {
+ std::string cf_name = column.name + "-" + std::to_string(i);
+ emplace_cf(column, i, cf_name, cf_opt);
+ }
+ }
+ }
+ existing_cfs.emplace_back("default", opt);
+
+ if (existing_cfs.size() != rocksdb_cfs.size()) {
+ std::vector<std::string> columns_from_stored;
+ sharding_def_to_columns(stored_sharding_def, columns_from_stored);
+ derr << __func__ << " extra columns in rocksdb. rocksdb columns = " << rocksdb_cfs
+ << " target columns = " << columns_from_stored << dendl;
+ return -EIO;
+ }
+ return 0;
+}
+
+std::ostream& operator<<(std::ostream& out, const RocksDBStore::ColumnFamily& cf)
+{
+ out << "(";
+ out << cf.name;
+ out << ",";
+ out << cf.shard_cnt;
+ out << ",";
+ out << cf.hash_l;
+ out << "-";
+ if (cf.hash_h != std::numeric_limits<uint32_t>::max()) {
+ out << cf.hash_h;
+ }
+ out << ",";
+ out << cf.options;
+ out << ")";
+ return out;
+}
+
+int RocksDBStore::do_open(ostream &out,
+ bool create_if_missing,
+ bool open_readonly,
+ const std::string& sharding_text)
+{
+ ceph_assert(!(create_if_missing && open_readonly));
+ rocksdb::Options opt;
+ int r = load_rocksdb_options(create_if_missing, opt);
+ if (r) {
+ dout(1) << __func__ << " load rocksdb options failed" << dendl;
+ return r;
+ }
+ rocksdb::Status status;
+ if (create_if_missing) {
+ status = rocksdb::DB::Open(opt, path, &db);
+ if (!status.ok()) {
+ derr << status.ToString() << dendl;
+ return -EINVAL;
+ }
+ r = apply_sharding(opt, sharding_text);
+ if (r < 0) {
+ return r;
+ }
+ default_cf = db->DefaultColumnFamily();
+ } else {
+ std::vector<rocksdb::ColumnFamilyDescriptor> existing_cfs;
+ std::vector<std::pair<size_t, RocksDBStore::ColumnFamily> > existing_cfs_shard;
+ std::vector<rocksdb::ColumnFamilyDescriptor> missing_cfs;
+ std::vector<std::pair<size_t, RocksDBStore::ColumnFamily> > missing_cfs_shard;
+
+ r = verify_sharding(opt,
+ existing_cfs, existing_cfs_shard,
+ missing_cfs, missing_cfs_shard);
+ if (r < 0) {
+ return r;
+ }
+ std::string sharding_recreate_text;
+ status = rocksdb::ReadFileToString(opt.env,
+ sharding_recreate,
+ &sharding_recreate_text);
+ bool recreate_mode = status.ok() && sharding_recreate_text == "1";
+
+ ceph_assert(!recreate_mode || !open_readonly);
+ if (recreate_mode == false && missing_cfs.size() != 0) {
+ // We do not accept when there are missing column families, except case that we are during resharding.
+ // We can get into this case if resharding was interrupted. It gives a chance to continue.
+ // Opening DB is only allowed in read-only mode.
+ if (open_readonly == false &&
+ std::find_if(missing_cfs.begin(), missing_cfs.end(),
+ [](const rocksdb::ColumnFamilyDescriptor& c) { return c.name == resharding_column_lock; }
+ ) != missing_cfs.end()) {
+ derr << __func__ << " missing column families: " << missing_cfs_shard << dendl;
+ return -EIO;
+ }
+ }
+
+ if (existing_cfs.empty()) {
+ // no column families
+ if (open_readonly) {
+ status = rocksdb::DB::OpenForReadOnly(opt, path, &db);
+ } else {
+ status = rocksdb::DB::Open(opt, path, &db);
+ }
+ if (!status.ok()) {
+ derr << status.ToString() << dendl;
+ return -EINVAL;
+ }
+ default_cf = db->DefaultColumnFamily();
+ } else {
+ std::vector<rocksdb::ColumnFamilyHandle*> handles;
+ if (open_readonly) {
+ status = rocksdb::DB::OpenForReadOnly(rocksdb::DBOptions(opt),
+ path, existing_cfs,
+ &handles, &db);
+ } else {
+ status = rocksdb::DB::Open(rocksdb::DBOptions(opt),
+ path, existing_cfs, &handles, &db);
+ }
+ if (!status.ok()) {
+ derr << status.ToString() << dendl;
+ return -EINVAL;
+ }
+ ceph_assert(existing_cfs.size() == existing_cfs_shard.size() + 1);
+ ceph_assert(handles.size() == existing_cfs.size());
+ dout(10) << __func__ << " existing_cfs=" << existing_cfs.size() << dendl;
+ for (size_t i = 0; i < existing_cfs_shard.size(); i++) {
+ add_column_family(existing_cfs_shard[i].second.name,
+ existing_cfs_shard[i].second.hash_l,
+ existing_cfs_shard[i].second.hash_h,
+ existing_cfs_shard[i].first,
+ handles[i]);
+ }
+ default_cf = handles[handles.size() - 1];
+ must_close_default_cf = true;
+
+ if (missing_cfs.size() > 0 &&
+ std::find_if(missing_cfs.begin(), missing_cfs.end(),
+ [](const rocksdb::ColumnFamilyDescriptor& c) { return c.name == resharding_column_lock; }
+ ) == missing_cfs.end())
+ {
+ dout(10) << __func__ << " missing_cfs=" << missing_cfs.size() << dendl;
+ ceph_assert(recreate_mode);
+ ceph_assert(missing_cfs.size() == missing_cfs_shard.size());
+ for (size_t i = 0; i < missing_cfs.size(); i++) {
+ rocksdb::ColumnFamilyHandle *cf;
+ status = db->CreateColumnFamily(missing_cfs[i].options, missing_cfs[i].name, &cf);
+ if (!status.ok()) {
+ derr << __func__ << " Failed to create rocksdb column family: "
+ << missing_cfs[i].name << dendl;
+ return -EINVAL;
+ }
+ add_column_family(missing_cfs_shard[i].second.name,
+ missing_cfs_shard[i].second.hash_l,
+ missing_cfs_shard[i].second.hash_h,
+ missing_cfs_shard[i].first,
+ cf);
+ }
+ opt.env->DeleteFile(sharding_recreate);
+ }
+ }
+ }
+ ceph_assert(default_cf != nullptr);
+
+ PerfCountersBuilder plb(cct, "rocksdb", l_rocksdb_first, l_rocksdb_last);
+ plb.add_time_avg(l_rocksdb_get_latency, "get_latency", "Get latency");
+ plb.add_time_avg(l_rocksdb_submit_latency, "submit_latency", "Submit Latency");
+ plb.add_time_avg(l_rocksdb_submit_sync_latency, "submit_sync_latency", "Submit Sync Latency");
+ plb.add_u64_counter(l_rocksdb_compact, "compact", "Compactions");
+ plb.add_u64_counter(l_rocksdb_compact_range, "compact_range", "Compactions by range");
+ plb.add_u64_counter(l_rocksdb_compact_queue_merge, "compact_queue_merge", "Mergings of ranges in compaction queue");
+ plb.add_u64(l_rocksdb_compact_queue_len, "compact_queue_len", "Length of compaction queue");
+ plb.add_time_avg(l_rocksdb_write_wal_time, "rocksdb_write_wal_time", "Rocksdb write wal time");
+ plb.add_time_avg(l_rocksdb_write_memtable_time, "rocksdb_write_memtable_time", "Rocksdb write memtable time");
+ plb.add_time_avg(l_rocksdb_write_delay_time, "rocksdb_write_delay_time", "Rocksdb write delay time");
+ plb.add_time_avg(l_rocksdb_write_pre_and_post_process_time,
+ "rocksdb_write_pre_and_post_time", "total time spent on writing a record, excluding write process");
+ logger = plb.create_perf_counters();
+ cct->get_perfcounters_collection()->add(logger);
+
+ if (compact_on_mount) {
+ derr << "Compacting rocksdb store..." << dendl;
+ compact();
+ derr << "Finished compacting rocksdb store" << dendl;
+ }
+ return 0;
+}
+
+int RocksDBStore::_test_init(const string& dir)
+{
+ rocksdb::Options options;
+ options.create_if_missing = true;
+ rocksdb::DB *db;
+ rocksdb::Status status = rocksdb::DB::Open(options, dir, &db);
+ delete db;
+ db = nullptr;
+ return status.ok() ? 0 : -EIO;
+}
+
+RocksDBStore::~RocksDBStore()
+{
+ close();
+ if (priv) {
+ delete static_cast<rocksdb::Env*>(priv);
+ }
+}
+
+void RocksDBStore::close()
+{
+ // stop compaction thread
+ compact_queue_lock.lock();
+ if (compact_thread.is_started()) {
+ dout(1) << __func__ << " waiting for compaction thread to stop" << dendl;
+ compact_queue_stop = true;
+ compact_queue_cond.notify_all();
+ compact_queue_lock.unlock();
+ compact_thread.join();
+ dout(1) << __func__ << " compaction thread to stopped" << dendl;
+ } else {
+ compact_queue_lock.unlock();
+ }
+
+ if (logger) {
+ cct->get_perfcounters_collection()->remove(logger);
+ delete logger;
+ logger = nullptr;
+ }
+
+ // Ensure db is destroyed before dependent db_cache and filterpolicy
+ for (auto& p : cf_handles) {
+ for (size_t i = 0; i < p.second.handles.size(); i++) {
+ db->DestroyColumnFamilyHandle(p.second.handles[i]);
+ }
+ }
+ cf_handles.clear();
+ if (must_close_default_cf) {
+ db->DestroyColumnFamilyHandle(default_cf);
+ must_close_default_cf = false;
+ }
+ default_cf = nullptr;
+ delete db;
+ db = nullptr;
+}
+
+int RocksDBStore::repair(std::ostream &out)
+{
+ rocksdb::Status status;
+ rocksdb::Options opt;
+ int r = load_rocksdb_options(false, opt);
+ if (r) {
+ dout(1) << __func__ << " load rocksdb options failed" << dendl;
+ out << "load rocksdb options failed" << std::endl;
+ return r;
+ }
+ //need to save sharding definition, repairDB will delete files it does not know
+ std::string stored_sharding_text;
+ status = opt.env->FileExists(sharding_def_file);
+ if (status.ok()) {
+ status = rocksdb::ReadFileToString(opt.env,
+ sharding_def_file,
+ &stored_sharding_text);
+ if (!status.ok()) {
+ stored_sharding_text.clear();
+ }
+ }
+ dout(10) << __func__ << " stored_sharding: " << stored_sharding_text << dendl;
+ status = rocksdb::RepairDB(path, opt);
+ bool repaired = status.ok();
+ if (!stored_sharding_text.empty()) {
+ //recreate markers even if repair failed
+ opt.env->CreateDir(sharding_def_dir);
+ status = rocksdb::WriteStringToFile(opt.env, stored_sharding_text,
+ sharding_def_file, true);
+ if (!status.ok()) {
+ derr << __func__ << " cannot write to " << sharding_def_file << dendl;
+ return -1;
+ }
+ status = rocksdb::WriteStringToFile(opt.env, "1",
+ sharding_recreate, true);
+ if (!status.ok()) {
+ derr << __func__ << " cannot write to " << sharding_recreate << dendl;
+ return -1;
+ }
+ // fiinalize sharding recreate
+ if (do_open(out, false, false)) {
+ derr << __func__ << " cannot finalize repair" << dendl;
+ return -1;
+ }
+ close();
+ }
+
+ if (repaired && status.ok()) {
+ return 0;
+ } else {
+ out << "repair rocksdb failed : " << status.ToString() << std::endl;
+ return -1;
+ }
+}
+
+void RocksDBStore::split_stats(const std::string &s, char delim, std::vector<std::string> &elems) {
+ std::stringstream ss;
+ ss.str(s);
+ std::string item;
+ while (std::getline(ss, item, delim)) {
+ elems.push_back(item);
+ }
+}
+
+bool RocksDBStore::get_property(
+ const std::string &property,
+ uint64_t *out)
+{
+ return db->GetIntProperty(property, out);
+}
+
+int64_t RocksDBStore::estimate_prefix_size(const string& prefix,
+ const string& key_prefix)
+{
+ uint64_t size = 0;
+ auto p_iter = cf_handles.find(prefix);
+ if (p_iter != cf_handles.end()) {
+ for (auto cf : p_iter->second.handles) {
+ uint64_t s = 0;
+ string start = key_prefix + string(1, '\x00');
+ string limit = key_prefix + string("\xff\xff\xff\xff");
+ rocksdb::Range r(start, limit);
+ db->GetApproximateSizes(cf, &r, 1, &s);
+ size += s;
+ }
+ } else {
+ string start = combine_strings(prefix , key_prefix);
+ string limit = combine_strings(prefix , key_prefix + "\xff\xff\xff\xff");
+ rocksdb::Range r(start, limit);
+ db->GetApproximateSizes(default_cf, &r, 1, &size);
+ }
+ return size;
+}
+
+void RocksDBStore::get_statistics(Formatter *f)
+{
+ if (!cct->_conf->rocksdb_perf) {
+ dout(20) << __func__ << " RocksDB perf is disabled, can't probe for stats"
+ << dendl;
+ return;
+ }
+
+ if (cct->_conf->rocksdb_collect_compaction_stats) {
+ std::string stat_str;
+ bool status = db->GetProperty("rocksdb.stats", &stat_str);
+ if (status) {
+ f->open_object_section("rocksdb_statistics");
+ f->dump_string("rocksdb_compaction_statistics", "");
+ vector<string> stats;
+ split_stats(stat_str, '\n', stats);
+ for (auto st :stats) {
+ f->dump_string("", st);
+ }
+ f->close_section();
+ }
+ }
+ if (cct->_conf->rocksdb_collect_extended_stats) {
+ if (dbstats) {
+ f->open_object_section("rocksdb_extended_statistics");
+ string stat_str = dbstats->ToString();
+ vector<string> stats;
+ split_stats(stat_str, '\n', stats);
+ f->dump_string("rocksdb_extended_statistics", "");
+ for (auto st :stats) {
+ f->dump_string(".", st);
+ }
+ f->close_section();
+ }
+ f->open_object_section("rocksdbstore_perf_counters");
+ logger->dump_formatted(f, false, false);
+ f->close_section();
+ }
+ if (cct->_conf->rocksdb_collect_memory_stats) {
+ f->open_object_section("rocksdb_memtable_statistics");
+ std::string str;
+ if (!bbt_opts.no_block_cache) {
+ str.append(stringify(bbt_opts.block_cache->GetUsage()));
+ f->dump_string("block_cache_usage", str.data());
+ str.clear();
+ str.append(stringify(bbt_opts.block_cache->GetPinnedUsage()));
+ f->dump_string("block_cache_pinned_blocks_usage", str);
+ str.clear();
+ }
+ db->GetProperty("rocksdb.cur-size-all-mem-tables", &str);
+ f->dump_string("rocksdb_memtable_usage", str);
+ str.clear();
+ db->GetProperty("rocksdb.estimate-table-readers-mem", &str);
+ f->dump_string("rocksdb_index_filter_blocks_usage", str);
+ f->close_section();
+ }
+}
+
+struct RocksDBStore::RocksWBHandler: public rocksdb::WriteBatch::Handler {
+ RocksWBHandler(const RocksDBStore& db) : db(db) {}
+ const RocksDBStore& db;
+ std::stringstream seen;
+ int num_seen = 0;
+
+ void dump(const char* op_name,
+ uint32_t column_family_id,
+ const rocksdb::Slice& key_in,
+ const rocksdb::Slice* value = nullptr) {
+ string prefix;
+ string key;
+ ssize_t size = value ? value->size() : -1;
+ seen << std::endl << op_name << "(";
+
+ if (column_family_id == 0) {
+ db.split_key(key_in, &prefix, &key);
+ } else {
+ auto it = db.cf_ids_to_prefix.find(column_family_id);
+ ceph_assert(it != db.cf_ids_to_prefix.end());
+ prefix = it->second;
+ key = key_in.ToString();
+ }
+ seen << " prefix = " << prefix;
+ seen << " key = " << pretty_binary_string(key);
+ if (size != -1)
+ seen << " value size = " << std::to_string(size);
+ seen << ")";
+ num_seen++;
+ }
+ void Put(const rocksdb::Slice& key,
+ const rocksdb::Slice& value) override {
+ dump("Put", 0, key, &value);
+ }
+ rocksdb::Status PutCF(uint32_t column_family_id, const rocksdb::Slice& key,
+ const rocksdb::Slice& value) override {
+ dump("PutCF", column_family_id, key, &value);
+ return rocksdb::Status::OK();
+ }
+ void SingleDelete(const rocksdb::Slice& key) override {
+ dump("SingleDelete", 0, key);
+ }
+ rocksdb::Status SingleDeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
+ dump("SingleDeleteCF", column_family_id, key);
+ return rocksdb::Status::OK();
+ }
+ void Delete(const rocksdb::Slice& key) override {
+ dump("Delete", 0, key);
+ }
+ rocksdb::Status DeleteCF(uint32_t column_family_id, const rocksdb::Slice& key) override {
+ dump("DeleteCF", column_family_id, key);
+ return rocksdb::Status::OK();
+ }
+ void Merge(const rocksdb::Slice& key,
+ const rocksdb::Slice& value) override {
+ dump("Merge", 0, key, &value);
+ }
+ rocksdb::Status MergeCF(uint32_t column_family_id, const rocksdb::Slice& key,
+ const rocksdb::Slice& value) override {
+ dump("MergeCF", column_family_id, key, &value);
+ return rocksdb::Status::OK();
+ }
+ bool Continue() override { return num_seen < 50; }
+};
+
+int RocksDBStore::submit_common(rocksdb::WriteOptions& woptions, KeyValueDB::Transaction t)
+{
+ // enable rocksdb breakdown
+ // considering performance overhead, default is disabled
+ if (cct->_conf->rocksdb_perf) {
+ rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
+ rocksdb::get_perf_context()->Reset();
+ }
+
+ RocksDBTransactionImpl * _t =
+ static_cast<RocksDBTransactionImpl *>(t.get());
+ woptions.disableWAL = disableWAL;
+ lgeneric_subdout(cct, rocksdb, 30) << __func__;
+ RocksWBHandler bat_txc(*this);
+ _t->bat.Iterate(&bat_txc);
+ *_dout << " Rocksdb transaction: " << bat_txc.seen.str() << dendl;
+
+ rocksdb::Status s = db->Write(woptions, &_t->bat);
+ if (!s.ok()) {
+ RocksWBHandler rocks_txc(*this);
+ _t->bat.Iterate(&rocks_txc);
+ derr << __func__ << " error: " << s.ToString() << " code = " << s.code()
+ << " Rocksdb transaction: " << rocks_txc.seen.str() << dendl;
+ }
+
+ if (cct->_conf->rocksdb_perf) {
+ utime_t write_memtable_time;
+ utime_t write_delay_time;
+ utime_t write_wal_time;
+ utime_t write_pre_and_post_process_time;
+ write_wal_time.set_from_double(
+ static_cast<double>(rocksdb::get_perf_context()->write_wal_time)/1000000000);
+ write_memtable_time.set_from_double(
+ static_cast<double>(rocksdb::get_perf_context()->write_memtable_time)/1000000000);
+ write_delay_time.set_from_double(
+ static_cast<double>(rocksdb::get_perf_context()->write_delay_time)/1000000000);
+ write_pre_and_post_process_time.set_from_double(
+ static_cast<double>(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000);
+ logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time);
+ logger->tinc(l_rocksdb_write_delay_time, write_delay_time);
+ logger->tinc(l_rocksdb_write_wal_time, write_wal_time);
+ logger->tinc(l_rocksdb_write_pre_and_post_process_time, write_pre_and_post_process_time);
+ }
+
+ return s.ok() ? 0 : -1;
+}
+
+int RocksDBStore::submit_transaction(KeyValueDB::Transaction t)
+{
+ utime_t start = ceph_clock_now();
+ rocksdb::WriteOptions woptions;
+ woptions.sync = false;
+
+ int result = submit_common(woptions, t);
+
+ utime_t lat = ceph_clock_now() - start;
+ logger->tinc(l_rocksdb_submit_latency, lat);
+
+ return result;
+}
+
+int RocksDBStore::submit_transaction_sync(KeyValueDB::Transaction t)
+{
+ utime_t start = ceph_clock_now();
+ rocksdb::WriteOptions woptions;
+ // if disableWAL, sync can't set
+ woptions.sync = !disableWAL;
+
+ int result = submit_common(woptions, t);
+
+ utime_t lat = ceph_clock_now() - start;
+ logger->tinc(l_rocksdb_submit_sync_latency, lat);
+
+ return result;
+}
+
+RocksDBStore::RocksDBTransactionImpl::RocksDBTransactionImpl(RocksDBStore *_db)
+{
+ db = _db;
+}
+
+void RocksDBStore::RocksDBTransactionImpl::put_bat(
+ rocksdb::WriteBatch& bat,
+ rocksdb::ColumnFamilyHandle *cf,
+ const string &key,
+ const bufferlist &to_set_bl)
+{
+ // bufferlist::c_str() is non-constant, so we can't call c_str()
+ if (to_set_bl.is_contiguous() && to_set_bl.length() > 0) {
+ bat.Put(cf,
+ rocksdb::Slice(key),
+ rocksdb::Slice(to_set_bl.buffers().front().c_str(),
+ to_set_bl.length()));
+ } else {
+ rocksdb::Slice key_slice(key);
+ vector<rocksdb::Slice> value_slices(to_set_bl.get_num_buffers());
+ bat.Put(cf,
+ rocksdb::SliceParts(&key_slice, 1),
+ prepare_sliceparts(to_set_bl, &value_slices));
+ }
+}
+
+void RocksDBStore::RocksDBTransactionImpl::set(
+ const string &prefix,
+ const string &k,
+ const bufferlist &to_set_bl)
+{
+ auto cf = db->get_cf_handle(prefix, k);
+ if (cf) {
+ put_bat(bat, cf, k, to_set_bl);
+ } else {
+ string key = combine_strings(prefix, k);
+ put_bat(bat, db->default_cf, key, to_set_bl);
+ }
+}
+
+void RocksDBStore::RocksDBTransactionImpl::set(
+ const string &prefix,
+ const char *k, size_t keylen,
+ const bufferlist &to_set_bl)
+{
+ auto cf = db->get_cf_handle(prefix, k, keylen);
+ if (cf) {
+ string key(k, keylen); // fixme?
+ put_bat(bat, cf, key, to_set_bl);
+ } else {
+ string key;
+ combine_strings(prefix, k, keylen, &key);
+ put_bat(bat, cf, key, to_set_bl);
+ }
+}
+
+void RocksDBStore::RocksDBTransactionImpl::rmkey(const string &prefix,
+ const string &k)
+{
+ auto cf = db->get_cf_handle(prefix, k);
+ if (cf) {
+ bat.Delete(cf, rocksdb::Slice(k));
+ } else {
+ bat.Delete(db->default_cf, combine_strings(prefix, k));
+ }
+}
+
+void RocksDBStore::RocksDBTransactionImpl::rmkey(const string &prefix,
+ const char *k,
+ size_t keylen)
+{
+ auto cf = db->get_cf_handle(prefix, k, keylen);
+ if (cf) {
+ bat.Delete(cf, rocksdb::Slice(k, keylen));
+ } else {
+ string key;
+ combine_strings(prefix, k, keylen, &key);
+ bat.Delete(db->default_cf, rocksdb::Slice(key));
+ }
+}
+
+void RocksDBStore::RocksDBTransactionImpl::rm_single_key(const string &prefix,
+ const string &k)
+{
+ auto cf = db->get_cf_handle(prefix, k);
+ if (cf) {
+ bat.SingleDelete(cf, k);
+ } else {
+ bat.SingleDelete(db->default_cf, combine_strings(prefix, k));
+ }
+}
+
+void RocksDBStore::RocksDBTransactionImpl::rmkeys_by_prefix(const string &prefix)
+{
+ auto p_iter = db->cf_handles.find(prefix);
+ if (p_iter == db->cf_handles.end()) {
+ uint64_t cnt = db->get_delete_range_threshold();
+ bat.SetSavePoint();
+ auto it = db->get_iterator(prefix);
+ for (it->seek_to_first(); it->valid() && (--cnt) != 0; it->next()) {
+ bat.Delete(db->default_cf, combine_strings(prefix, it->key()));
+ }
+ if (cnt == 0) {
+ bat.RollbackToSavePoint();
+ string endprefix = prefix;
+ endprefix.push_back('\x01');
+ bat.DeleteRange(db->default_cf,
+ combine_strings(prefix, string()),
+ combine_strings(endprefix, string()));
+ } else {
+ bat.PopSavePoint();
+ }
+ } else {
+ ceph_assert(p_iter->second.handles.size() >= 1);
+ for (auto cf : p_iter->second.handles) {
+ uint64_t cnt = db->get_delete_range_threshold();
+ bat.SetSavePoint();
+ auto it = db->new_shard_iterator(cf);
+ for (it->seek_to_first(); it->valid() && (--cnt) != 0; it->next()) {
+ bat.Delete(cf, it->key());
+ }
+ if (cnt == 0) {
+ bat.RollbackToSavePoint();
+ string endprefix = "\xff\xff\xff\xff"; // FIXME: this is cheating...
+ bat.DeleteRange(cf, string(), endprefix);
+ } else {
+ bat.PopSavePoint();
+ }
+ }
+ }
+}
+
+void RocksDBStore::RocksDBTransactionImpl::rm_range_keys(const string &prefix,
+ const string &start,
+ const string &end)
+{
+ ldout(db->cct, 10) << __func__
+ << " enter prefix=" << prefix
+ << " start=" << pretty_binary_string(start)
+ << " end=" << pretty_binary_string(end) << dendl;
+ auto p_iter = db->cf_handles.find(prefix);
+ uint64_t cnt = db->get_delete_range_threshold();
+ if (p_iter == db->cf_handles.end()) {
+ uint64_t cnt0 = cnt;
+ bat.SetSavePoint();
+ auto it = db->get_iterator(prefix);
+ for (it->lower_bound(start);
+ it->valid() && db->comparator->Compare(it->key(), end) < 0 && (--cnt) != 0;
+ it->next()) {
+ bat.Delete(db->default_cf, combine_strings(prefix, it->key()));
+ }
+ ldout(db->cct, 15) << __func__
+ << " count = " << cnt0 - cnt
+ << dendl;
+ if (cnt == 0) {
+ ldout(db->cct, 10) << __func__ << " p_iter == end(), resorting to DeleteRange"
+ << dendl;
+ bat.RollbackToSavePoint();
+ bat.DeleteRange(db->default_cf,
+ rocksdb::Slice(combine_strings(prefix, start)),
+ rocksdb::Slice(combine_strings(prefix, end)));
+ } else {
+ bat.PopSavePoint();
+ }
+ } else if (cnt == 0) {
+ ceph_assert(p_iter->second.handles.size() >= 1);
+ for (auto cf : p_iter->second.handles) {
+ ldout(db->cct, 10) << __func__ << " p_iter != end(), resorting to DeleteRange"
+ << dendl;
+ bat.DeleteRange(cf, rocksdb::Slice(start), rocksdb::Slice(end));
+ }
+ } else {
+ auto bounds = KeyValueDB::IteratorBounds();
+ bounds.lower_bound = start;
+ bounds.upper_bound = end;
+ ceph_assert(p_iter->second.handles.size() >= 1);
+ for (auto cf : p_iter->second.handles) {
+ cnt = db->get_delete_range_threshold();
+ uint64_t cnt0 = cnt;
+ bat.SetSavePoint();
+ auto it = db->new_shard_iterator(cf, prefix, bounds);
+ for (it->lower_bound(start);
+ it->valid() && (--cnt) != 0;
+ it->next()) {
+ bat.Delete(cf, it->key());
+ }
+ ldout(db->cct, 10) << __func__
+ << " count = " << cnt0 - cnt
+ << dendl;
+ if (cnt == 0) {
+ ldout(db->cct, 10) << __func__ << " p_iter != end(), resorting to DeleteRange"
+ << dendl;
+ bat.RollbackToSavePoint();
+ bat.DeleteRange(cf, rocksdb::Slice(start), rocksdb::Slice(end));
+ } else {
+ bat.PopSavePoint();
+ }
+ }
+ }
+ ldout(db->cct, 10) << __func__ << " end" << dendl;
+}
+
+void RocksDBStore::RocksDBTransactionImpl::merge(
+ const string &prefix,
+ const string &k,
+ const bufferlist &to_set_bl)
+{
+ auto cf = db->get_cf_handle(prefix, k);
+ if (cf) {
+ // bufferlist::c_str() is non-constant, so we can't call c_str()
+ if (to_set_bl.is_contiguous() && to_set_bl.length() > 0) {
+ bat.Merge(
+ cf,
+ rocksdb::Slice(k),
+ rocksdb::Slice(to_set_bl.buffers().front().c_str(), to_set_bl.length()));
+ } else {
+ // make a copy
+ rocksdb::Slice key_slice(k);
+ vector<rocksdb::Slice> value_slices(to_set_bl.get_num_buffers());
+ bat.Merge(cf, rocksdb::SliceParts(&key_slice, 1),
+ prepare_sliceparts(to_set_bl, &value_slices));
+ }
+ } else {
+ string key = combine_strings(prefix, k);
+ // bufferlist::c_str() is non-constant, so we can't call c_str()
+ if (to_set_bl.is_contiguous() && to_set_bl.length() > 0) {
+ bat.Merge(
+ db->default_cf,
+ rocksdb::Slice(key),
+ rocksdb::Slice(to_set_bl.buffers().front().c_str(), to_set_bl.length()));
+ } else {
+ // make a copy
+ rocksdb::Slice key_slice(key);
+ vector<rocksdb::Slice> value_slices(to_set_bl.get_num_buffers());
+ bat.Merge(
+ db->default_cf,
+ rocksdb::SliceParts(&key_slice, 1),
+ prepare_sliceparts(to_set_bl, &value_slices));
+ }
+ }
+}
+
+int RocksDBStore::get(
+ const string &prefix,
+ const std::set<string> &keys,
+ std::map<string, bufferlist> *out)
+{
+ rocksdb::PinnableSlice value;
+ utime_t start = ceph_clock_now();
+ if (cf_handles.count(prefix) > 0) {
+ for (auto& key : keys) {
+ auto cf_handle = get_cf_handle(prefix, key);
+ auto status = db->Get(rocksdb::ReadOptions(),
+ cf_handle,
+ rocksdb::Slice(key),
+ &value);
+ if (status.ok()) {
+ (*out)[key].append(value.data(), value.size());
+ } else if (status.IsIOError()) {
+ ceph_abort_msg(status.getState());
+ }
+ value.Reset();
+ }
+ } else {
+ for (auto& key : keys) {
+ string k = combine_strings(prefix, key);
+ auto status = db->Get(rocksdb::ReadOptions(),
+ default_cf,
+ rocksdb::Slice(k),
+ &value);
+ if (status.ok()) {
+ (*out)[key].append(value.data(), value.size());
+ } else if (status.IsIOError()) {
+ ceph_abort_msg(status.getState());
+ }
+ value.Reset();
+ }
+ }
+ utime_t lat = ceph_clock_now() - start;
+ logger->tinc(l_rocksdb_get_latency, lat);
+ return 0;
+}
+
+int RocksDBStore::get(
+ const string &prefix,
+ const string &key,
+ bufferlist *out)
+{
+ ceph_assert(out && (out->length() == 0));
+ utime_t start = ceph_clock_now();
+ int r = 0;
+ rocksdb::PinnableSlice value;
+ rocksdb::Status s;
+ auto cf = get_cf_handle(prefix, key);
+ if (cf) {
+ s = db->Get(rocksdb::ReadOptions(),
+ cf,
+ rocksdb::Slice(key),
+ &value);
+ } else {
+ string k = combine_strings(prefix, key);
+ s = db->Get(rocksdb::ReadOptions(),
+ default_cf,
+ rocksdb::Slice(k),
+ &value);
+ }
+ if (s.ok()) {
+ out->append(value.data(), value.size());
+ } else if (s.IsNotFound()) {
+ r = -ENOENT;
+ } else {
+ ceph_abort_msg(s.getState());
+ }
+ utime_t lat = ceph_clock_now() - start;
+ logger->tinc(l_rocksdb_get_latency, lat);
+ return r;
+}
+
+int RocksDBStore::get(
+ const string& prefix,
+ const char *key,
+ size_t keylen,
+ bufferlist *out)
+{
+ ceph_assert(out && (out->length() == 0));
+ utime_t start = ceph_clock_now();
+ int r = 0;
+ rocksdb::PinnableSlice value;
+ rocksdb::Status s;
+ auto cf = get_cf_handle(prefix, key, keylen);
+ if (cf) {
+ s = db->Get(rocksdb::ReadOptions(),
+ cf,
+ rocksdb::Slice(key, keylen),
+ &value);
+ } else {
+ string k;
+ combine_strings(prefix, key, keylen, &k);
+ s = db->Get(rocksdb::ReadOptions(),
+ default_cf,
+ rocksdb::Slice(k),
+ &value);
+ }
+ if (s.ok()) {
+ out->append(value.data(), value.size());
+ } else if (s.IsNotFound()) {
+ r = -ENOENT;
+ } else {
+ ceph_abort_msg(s.getState());
+ }
+ utime_t lat = ceph_clock_now() - start;
+ logger->tinc(l_rocksdb_get_latency, lat);
+ return r;
+}
+
+int RocksDBStore::split_key(rocksdb::Slice in, string *prefix, string *key)
+{
+ size_t prefix_len = 0;
+
+ // Find separator inside Slice
+ char* separator = (char*) memchr(in.data(), 0, in.size());
+ if (separator == NULL)
+ return -EINVAL;
+ prefix_len = size_t(separator - in.data());
+ if (prefix_len >= in.size())
+ return -EINVAL;
+
+ // Fetch prefix and/or key directly from Slice
+ if (prefix)
+ *prefix = string(in.data(), prefix_len);
+ if (key)
+ *key = string(separator+1, in.size()-prefix_len-1);
+ return 0;
+}
+
+void RocksDBStore::compact()
+{
+ logger->inc(l_rocksdb_compact);
+ rocksdb::CompactRangeOptions options;
+ db->CompactRange(options, default_cf, nullptr, nullptr);
+ for (auto cf : cf_handles) {
+ for (auto shard_cf : cf.second.handles) {
+ db->CompactRange(
+ options,
+ shard_cf,
+ nullptr, nullptr);
+ }
+ }
+}
+
+void RocksDBStore::compact_thread_entry()
+{
+ std::unique_lock l{compact_queue_lock};
+ dout(10) << __func__ << " enter" << dendl;
+ while (!compact_queue_stop) {
+ if (!compact_queue.empty()) {
+ auto range = compact_queue.front();
+ compact_queue.pop_front();
+ logger->set(l_rocksdb_compact_queue_len, compact_queue.size());
+ l.unlock();
+ logger->inc(l_rocksdb_compact_range);
+ if (range.first.empty() && range.second.empty()) {
+ compact();
+ } else {
+ compact_range(range.first, range.second);
+ }
+ l.lock();
+ continue;
+ }
+ dout(10) << __func__ << " waiting" << dendl;
+ compact_queue_cond.wait(l);
+ }
+ dout(10) << __func__ << " exit" << dendl;
+}
+
+void RocksDBStore::compact_range_async(const string& start, const string& end)
+{
+ std::lock_guard l(compact_queue_lock);
+
+ // try to merge adjacent ranges. this is O(n), but the queue should
+ // be short. note that we do not cover all overlap cases and merge
+ // opportunities here, but we capture the ones we currently need.
+ list< pair<string,string> >::iterator p = compact_queue.begin();
+ while (p != compact_queue.end()) {
+ if (p->first == start && p->second == end) {
+ // dup; no-op
+ return;
+ }
+ if (start <= p->first && p->first <= end) {
+ // new region crosses start of existing range
+ // select right bound that is bigger
+ compact_queue.push_back(make_pair(start, end > p->second ? end : p->second));
+ compact_queue.erase(p);
+ logger->inc(l_rocksdb_compact_queue_merge);
+ break;
+ }
+ if (start <= p->second && p->second <= end) {
+ // new region crosses end of existing range
+ //p->first < p->second and p->second <= end, so p->first <= end.
+ //But we break if previous condition, so start > p->first.
+ compact_queue.push_back(make_pair(p->first, end));
+ compact_queue.erase(p);
+ logger->inc(l_rocksdb_compact_queue_merge);
+ break;
+ }
+ ++p;
+ }
+ if (p == compact_queue.end()) {
+ // no merge, new entry.
+ compact_queue.push_back(make_pair(start, end));
+ logger->set(l_rocksdb_compact_queue_len, compact_queue.size());
+ }
+ compact_queue_cond.notify_all();
+ if (!compact_thread.is_started()) {
+ compact_thread.create("rstore_compact");
+ }
+}
+bool RocksDBStore::check_omap_dir(string &omap_dir)
+{
+ rocksdb::Options options;
+ options.create_if_missing = true;
+ rocksdb::DB *db;
+ rocksdb::Status status = rocksdb::DB::Open(options, omap_dir, &db);
+ delete db;
+ db = nullptr;
+ return status.ok();
+}
+
+void RocksDBStore::compact_range(const string& start, const string& end)
+{
+ rocksdb::CompactRangeOptions options;
+ rocksdb::Slice cstart(start);
+ rocksdb::Slice cend(end);
+ string prefix_start, key_start;
+ string prefix_end, key_end;
+ string key_highest = "\xff\xff\xff\xff"; //cheating
+ string key_lowest = "";
+
+ auto compact_range = [&] (const decltype(cf_handles)::iterator column_it,
+ const std::string& start,
+ const std::string& end) {
+ rocksdb::Slice cstart(start);
+ rocksdb::Slice cend(end);
+ for (const auto& shard_it : column_it->second.handles) {
+ db->CompactRange(options, shard_it, &cstart, &cend);
+ }
+ };
+ db->CompactRange(options, default_cf, &cstart, &cend);
+ split_key(cstart, &prefix_start, &key_start);
+ split_key(cend, &prefix_end, &key_end);
+ if (prefix_start == prefix_end) {
+ const auto& column = cf_handles.find(prefix_start);
+ if (column != cf_handles.end()) {
+ compact_range(column, key_start, key_end);
+ }
+ } else {
+ auto column = cf_handles.find(prefix_start);
+ if (column != cf_handles.end()) {
+ compact_range(column, key_start, key_highest);
+ ++column;
+ }
+ const auto& column_end = cf_handles.find(prefix_end);
+ while (column != column_end) {
+ compact_range(column, key_lowest, key_highest);
+ column++;
+ }
+ if (column != cf_handles.end()) {
+ compact_range(column, key_lowest, key_end);
+ }
+ }
+}
+
+RocksDBStore::RocksDBWholeSpaceIteratorImpl::~RocksDBWholeSpaceIteratorImpl()
+{
+ delete dbiter;
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::seek_to_first()
+{
+ dbiter->SeekToFirst();
+ ceph_assert(!dbiter->status().IsIOError());
+ return dbiter->status().ok() ? 0 : -1;
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::seek_to_first(const string &prefix)
+{
+ rocksdb::Slice slice_prefix(prefix);
+ dbiter->Seek(slice_prefix);
+ ceph_assert(!dbiter->status().IsIOError());
+ return dbiter->status().ok() ? 0 : -1;
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::seek_to_last()
+{
+ dbiter->SeekToLast();
+ ceph_assert(!dbiter->status().IsIOError());
+ return dbiter->status().ok() ? 0 : -1;
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::seek_to_last(const string &prefix)
+{
+ string limit = past_prefix(prefix);
+ rocksdb::Slice slice_limit(limit);
+ dbiter->Seek(slice_limit);
+
+ if (!dbiter->Valid()) {
+ dbiter->SeekToLast();
+ } else {
+ dbiter->Prev();
+ }
+ return dbiter->status().ok() ? 0 : -1;
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::upper_bound(const string &prefix, const string &after)
+{
+ lower_bound(prefix, after);
+ if (valid()) {
+ pair<string,string> key = raw_key();
+ if (key.first == prefix && key.second == after)
+ next();
+ }
+ return dbiter->status().ok() ? 0 : -1;
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::lower_bound(const string &prefix, const string &to)
+{
+ string bound = combine_strings(prefix, to);
+ rocksdb::Slice slice_bound(bound);
+ dbiter->Seek(slice_bound);
+ return dbiter->status().ok() ? 0 : -1;
+}
+bool RocksDBStore::RocksDBWholeSpaceIteratorImpl::valid()
+{
+ return dbiter->Valid();
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::next()
+{
+ if (valid()) {
+ dbiter->Next();
+ }
+ ceph_assert(!dbiter->status().IsIOError());
+ return dbiter->status().ok() ? 0 : -1;
+}
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::prev()
+{
+ if (valid()) {
+ dbiter->Prev();
+ }
+ ceph_assert(!dbiter->status().IsIOError());
+ return dbiter->status().ok() ? 0 : -1;
+}
+string RocksDBStore::RocksDBWholeSpaceIteratorImpl::key()
+{
+ string out_key;
+ split_key(dbiter->key(), 0, &out_key);
+ return out_key;
+}
+pair<string,string> RocksDBStore::RocksDBWholeSpaceIteratorImpl::raw_key()
+{
+ string prefix, key;
+ split_key(dbiter->key(), &prefix, &key);
+ return make_pair(prefix, key);
+}
+
+bool RocksDBStore::RocksDBWholeSpaceIteratorImpl::raw_key_is_prefixed(const string &prefix) {
+ // Look for "prefix\0" right in rocksb::Slice
+ rocksdb::Slice key = dbiter->key();
+ if ((key.size() > prefix.length()) && (key[prefix.length()] == '\0')) {
+ return memcmp(key.data(), prefix.c_str(), prefix.length()) == 0;
+ } else {
+ return false;
+ }
+}
+
+bufferlist RocksDBStore::RocksDBWholeSpaceIteratorImpl::value()
+{
+ return to_bufferlist(dbiter->value());
+}
+
+size_t RocksDBStore::RocksDBWholeSpaceIteratorImpl::key_size()
+{
+ return dbiter->key().size();
+}
+
+size_t RocksDBStore::RocksDBWholeSpaceIteratorImpl::value_size()
+{
+ return dbiter->value().size();
+}
+
+bufferptr RocksDBStore::RocksDBWholeSpaceIteratorImpl::value_as_ptr()
+{
+ rocksdb::Slice val = dbiter->value();
+ return bufferptr(val.data(), val.size());
+}
+
+int RocksDBStore::RocksDBWholeSpaceIteratorImpl::status()
+{
+ return dbiter->status().ok() ? 0 : -1;
+}
+
+string RocksDBStore::past_prefix(const string &prefix)
+{
+ string limit = prefix;
+ limit.push_back(1);
+ return limit;
+}
+
+class CFIteratorImpl : public KeyValueDB::IteratorImpl {
+protected:
+ string prefix;
+ rocksdb::Iterator *dbiter;
+ const KeyValueDB::IteratorBounds bounds;
+ const rocksdb::Slice iterate_lower_bound;
+ const rocksdb::Slice iterate_upper_bound;
+public:
+ explicit CFIteratorImpl(const RocksDBStore* db,
+ const std::string& p,
+ rocksdb::ColumnFamilyHandle* cf,
+ KeyValueDB::IteratorBounds bounds_)
+ : prefix(p), bounds(std::move(bounds_)),
+ iterate_lower_bound(make_slice(bounds.lower_bound)),
+ iterate_upper_bound(make_slice(bounds.upper_bound))
+ {
+ auto options = rocksdb::ReadOptions();
+ if (db->cct->_conf->osd_rocksdb_iterator_bounds_enabled) {
+ if (bounds.lower_bound) {
+ options.iterate_lower_bound = &iterate_lower_bound;
+ }
+ if (bounds.upper_bound) {
+ options.iterate_upper_bound = &iterate_upper_bound;
+ }
+ }
+ dbiter = db->db->NewIterator(options, cf);
+ }
+ ~CFIteratorImpl() {
+ delete dbiter;
+ }
+
+ int seek_to_first() override {
+ dbiter->SeekToFirst();
+ return dbiter->status().ok() ? 0 : -1;
+ }
+ int seek_to_last() override {
+ dbiter->SeekToLast();
+ return dbiter->status().ok() ? 0 : -1;
+ }
+ int upper_bound(const string &after) override {
+ lower_bound(after);
+ if (valid() && (key() == after)) {
+ next();
+ }
+ return dbiter->status().ok() ? 0 : -1;
+ }
+ int lower_bound(const string &to) override {
+ rocksdb::Slice slice_bound(to);
+ dbiter->Seek(slice_bound);
+ return dbiter->status().ok() ? 0 : -1;
+ }
+ int next() override {
+ if (valid()) {
+ dbiter->Next();
+ }
+ return dbiter->status().ok() ? 0 : -1;
+ }
+ int prev() override {
+ if (valid()) {
+ dbiter->Prev();
+ }
+ return dbiter->status().ok() ? 0 : -1;
+ }
+ bool valid() override {
+ return dbiter->Valid();
+ }
+ string key() override {
+ return dbiter->key().ToString();
+ }
+ std::pair<std::string, std::string> raw_key() override {
+ return make_pair(prefix, key());
+ }
+ bufferlist value() override {
+ return to_bufferlist(dbiter->value());
+ }
+ bufferptr value_as_ptr() override {
+ rocksdb::Slice val = dbiter->value();
+ return bufferptr(val.data(), val.size());
+ }
+ int status() override {
+ return dbiter->status().ok() ? 0 : -1;
+ }
+};
+
+
+//merge column iterators and rest iterator
+class WholeMergeIteratorImpl : public KeyValueDB::WholeSpaceIteratorImpl {
+private:
+ RocksDBStore* db;
+ KeyValueDB::WholeSpaceIterator main;
+ std::map<std::string, KeyValueDB::Iterator> shards;
+ std::map<std::string, KeyValueDB::Iterator>::iterator current_shard;
+ enum {on_main, on_shard} smaller;
+
+public:
+ WholeMergeIteratorImpl(RocksDBStore* db)
+ : db(db)
+ , main(db->get_default_cf_iterator())
+ {
+ for (auto& e : db->cf_handles) {
+ shards.emplace(e.first, db->get_iterator(e.first));
+ }
+ }
+
+ // returns true if value in main is smaller then in shards
+ // invalid is larger then actual value
+ bool is_main_smaller() {
+ if (main->valid()) {
+ if (current_shard != shards.end()) {
+ auto main_rk = main->raw_key();
+ ceph_assert(current_shard->second->valid());
+ auto shards_rk = current_shard->second->raw_key();
+ if (main_rk.first < shards_rk.first)
+ return true;
+ if (main_rk.first > shards_rk.first)
+ return false;
+ return main_rk.second < shards_rk.second;
+ } else {
+ return true;
+ }
+ } else {
+ if (current_shard != shards.end()) {
+ return false;
+ } else {
+ //this means that neither is valid
+ //we select main to be smaller, so valid() will signal properly
+ return true;
+ }
+ }
+ }
+
+ int seek_to_first() override {
+ int r0 = main->seek_to_first();
+ int r1 = 0;
+ // find first shard that has some data
+ current_shard = shards.begin();
+ while (current_shard != shards.end()) {
+ r1 = current_shard->second->seek_to_first();
+ if (r1 != 0 || current_shard->second->valid()) {
+ //this is the first shard that will yield some keys
+ break;
+ }
+ ++current_shard;
+ }
+ smaller = is_main_smaller() ? on_main : on_shard;
+ return r0 == 0 && r1 == 0 ? 0 : -1;
+ }
+
+ int seek_to_first(const std::string &prefix) override {
+ int r0 = main->seek_to_first(prefix);
+ int r1 = 0;
+ // find first shard that has some data
+ current_shard = shards.lower_bound(prefix);
+ while (current_shard != shards.end()) {
+ r1 = current_shard->second->seek_to_first();
+ if (r1 != 0 || current_shard->second->valid()) {
+ //this is the first shard that will yield some keys
+ break;
+ }
+ ++current_shard;
+ }
+ smaller = is_main_smaller() ? on_main : on_shard;
+ return r0 == 0 && r1 == 0 ? 0 : -1;
+ };
+
+ int seek_to_last() override {
+ int r0 = main->seek_to_last();
+ int r1 = 0;
+ r1 = shards_seek_to_last();
+ //if we have 2 candidates, we need to select
+ if (main->valid()) {
+ if (shards_valid()) {
+ if (is_main_smaller()) {
+ smaller = on_shard;
+ main->next();
+ } else {
+ smaller = on_main;
+ shards_next();
+ }
+ } else {
+ smaller = on_main;
+ }
+ } else {
+ if (shards_valid()) {
+ smaller = on_shard;
+ } else {
+ smaller = on_main;
+ }
+ }
+ return r0 == 0 && r1 == 0 ? 0 : -1;
+ }
+
+ int seek_to_last(const std::string &prefix) override {
+ int r0 = main->seek_to_last(prefix);
+ int r1 = 0;
+ // find last shard that has some data
+ bool found = false;
+ current_shard = shards.lower_bound(prefix);
+ while (current_shard != shards.begin()) {
+ r1 = current_shard->second->seek_to_last();
+ if (r1 != 0)
+ break;
+ if (current_shard->second->valid()) {
+ found = true;
+ break;
+ }
+ }
+ //if we have 2 candidates, we need to select
+ if (main->valid() && found) {
+ if (is_main_smaller()) {
+ main->next();
+ } else {
+ shards_next();
+ }
+ }
+ if (!found) {
+ //set shards state that properly represents eof
+ current_shard = shards.end();
+ }
+ smaller = is_main_smaller() ? on_main : on_shard;
+ return r0 == 0 && r1 == 0 ? 0 : -1;
+ }
+
+ int upper_bound(const std::string &prefix, const std::string &after) override {
+ int r0 = main->upper_bound(prefix, after);
+ int r1 = 0;
+ if (r0 != 0)
+ return r0;
+ current_shard = shards.lower_bound(prefix);
+ if (current_shard != shards.end()) {
+ bool located = false;
+ if (current_shard->first == prefix) {
+ r1 = current_shard->second->upper_bound(after);
+ if (r1 != 0)
+ return r1;
+ if (current_shard->second->valid()) {
+ located = true;
+ }
+ }
+ if (!located) {
+ while (current_shard != shards.end()) {
+ r1 = current_shard->second->seek_to_first();
+ if (r1 != 0)
+ return r1;
+ if (current_shard->second->valid())
+ break;
+ ++current_shard;
+ }
+ }
+ }
+ smaller = is_main_smaller() ? on_main : on_shard;
+ return 0;
+ }
+
+ int lower_bound(const std::string &prefix, const std::string &to) override {
+ int r0 = main->lower_bound(prefix, to);
+ int r1 = 0;
+ if (r0 != 0)
+ return r0;
+ current_shard = shards.lower_bound(prefix);
+ if (current_shard != shards.end()) {
+ bool located = false;
+ if (current_shard->first == prefix) {
+ r1 = current_shard->second->lower_bound(to);
+ if (r1 != 0)
+ return r1;
+ if (current_shard->second->valid()) {
+ located = true;
+ }
+ }
+ if (!located) {
+ while (current_shard != shards.end()) {
+ r1 = current_shard->second->seek_to_first();
+ if (r1 != 0)
+ return r1;
+ if (current_shard->second->valid())
+ break;
+ ++current_shard;
+ }
+ }
+ }
+ smaller = is_main_smaller() ? on_main : on_shard;
+ return 0;
+ }
+
+ bool valid() override {
+ if (smaller == on_main) {
+ return main->valid();
+ } else {
+ if (current_shard == shards.end())
+ return false;
+ return current_shard->second->valid();
+ }
+ };
+
+ int next() override {
+ int r;
+ if (smaller == on_main) {
+ r = main->next();
+ } else {
+ r = shards_next();
+ }
+ if (r != 0)
+ return r;
+ smaller = is_main_smaller() ? on_main : on_shard;
+ return 0;
+ }
+
+ int prev() override {
+ int r;
+ bool main_was_valid = false;
+ if (main->valid()) {
+ main_was_valid = true;
+ r = main->prev();
+ } else {
+ r = main->seek_to_last();
+ }
+ if (r != 0)
+ return r;
+
+ bool shards_was_valid = false;
+ if (shards_valid()) {
+ shards_was_valid = true;
+ r = shards_prev();
+ } else {
+ r = shards_seek_to_last();
+ }
+ if (r != 0)
+ return r;
+
+ if (!main->valid() && !shards_valid()) {
+ //end, no previous. set marker so valid() can work
+ smaller = on_main;
+ return 0;
+ }
+
+ //if 1 is valid, select it
+ //if 2 are valid select larger and advance the other
+ if (main->valid()) {
+ if (shards_valid()) {
+ if (is_main_smaller()) {
+ smaller = on_shard;
+ if (main_was_valid) {
+ if (main->valid()) {
+ r = main->next();
+ } else {
+ r = main->seek_to_first();
+ }
+ } else {
+ //if we have resurrected main, kill it
+ if (main->valid()) {
+ main->next();
+ }
+ }
+ } else {
+ smaller = on_main;
+ if (shards_was_valid) {
+ if (shards_valid()) {
+ r = shards_next();
+ } else {
+ r = shards_seek_to_first();
+ }
+ } else {
+ //if we have resurected shards, kill it
+ if (shards_valid()) {
+ shards_next();
+ }
+ }
+ }
+ } else {
+ smaller = on_main;
+ r = shards_seek_to_first();
+ }
+ } else {
+ smaller = on_shard;
+ r = main->seek_to_first();
+ }
+ return r;
+ }
+
+ std::string key() override
+ {
+ if (smaller == on_main) {
+ return main->key();
+ } else {
+ return current_shard->second->key();
+ }
+ }
+
+ std::pair<std::string,std::string> raw_key() override
+ {
+ if (smaller == on_main) {
+ return main->raw_key();
+ } else {
+ return { current_shard->first, current_shard->second->key() };
+ }
+ }
+
+ bool raw_key_is_prefixed(const std::string &prefix) override
+ {
+ if (smaller == on_main) {
+ return main->raw_key_is_prefixed(prefix);
+ } else {
+ return current_shard->first == prefix;
+ }
+ }
+
+ ceph::buffer::list value() override
+ {
+ if (smaller == on_main) {
+ return main->value();
+ } else {
+ return current_shard->second->value();
+ }
+ }
+
+ int status() override
+ {
+ //because we already had to inspect key, it must be ok
+ return 0;
+ }
+
+ size_t key_size() override
+ {
+ if (smaller == on_main) {
+ return main->key_size();
+ } else {
+ return current_shard->second->key().size();
+ }
+ }
+ size_t value_size() override
+ {
+ if (smaller == on_main) {
+ return main->value_size();
+ } else {
+ return current_shard->second->value().length();
+ }
+ }
+
+ int shards_valid() {
+ if (current_shard == shards.end())
+ return false;
+ return current_shard->second->valid();
+ }
+
+ int shards_next() {
+ if (current_shard == shards.end()) {
+ //illegal to next() on !valid()
+ return -1;
+ }
+ int r = 0;
+ r = current_shard->second->next();
+ if (r != 0)
+ return r;
+ if (current_shard->second->valid())
+ return 0;
+ //current shard exhaused, search for key
+ ++current_shard;
+ while (current_shard != shards.end()) {
+ r = current_shard->second->seek_to_first();
+ if (r != 0)
+ return r;
+ if (current_shard->second->valid())
+ break;
+ ++current_shard;
+ }
+ //either we found key or not, but it is success
+ return 0;
+ }
+
+ int shards_prev() {
+ if (current_shard == shards.end()) {
+ //illegal to prev() on !valid()
+ return -1;
+ }
+ int r = current_shard->second->prev();
+ while (r == 0) {
+ if (current_shard->second->valid()) {
+ break;
+ }
+ if (current_shard == shards.begin()) {
+ //we have reached pre-first element
+ //this makes it !valid(), but guarantees next() moves to first element
+ break;
+ }
+ --current_shard;
+ r = current_shard->second->seek_to_last();
+ }
+ return r;
+ }
+
+ int shards_seek_to_last() {
+ int r = 0;
+ current_shard = shards.end();
+ if (current_shard == shards.begin()) {
+ //no shards at all
+ return 0;
+ }
+ while (current_shard != shards.begin()) {
+ --current_shard;
+ r = current_shard->second->seek_to_last();
+ if (r != 0)
+ return r;
+ if (current_shard->second->valid()) {
+ return 0;
+ }
+ }
+ //no keys at all
+ current_shard = shards.end();
+ return r;
+ }
+
+ int shards_seek_to_first() {
+ int r = 0;
+ current_shard = shards.begin();
+ while (current_shard != shards.end()) {
+ r = current_shard->second->seek_to_first();
+ if (r != 0)
+ break;
+ if (current_shard->second->valid()) {
+ //this is the first shard that will yield some keys
+ break;
+ }
+ ++current_shard;
+ }
+ return r;
+ }
+};
+
+class ShardMergeIteratorImpl : public KeyValueDB::IteratorImpl {
+private:
+ struct KeyLess {
+ private:
+ const rocksdb::Comparator* comparator;
+ public:
+ KeyLess(const rocksdb::Comparator* comparator) : comparator(comparator) { };
+
+ bool operator()(rocksdb::Iterator* a, rocksdb::Iterator* b) const
+ {
+ if (a->Valid()) {
+ if (b->Valid()) {
+ return comparator->Compare(a->key(), b->key()) < 0;
+ } else {
+ return true;
+ }
+ } else {
+ if (b->Valid()) {
+ return false;
+ } else {
+ return false;
+ }
+ }
+ }
+ };
+
+ const RocksDBStore* db;
+ KeyLess keyless;
+ string prefix;
+ const KeyValueDB::IteratorBounds bounds;
+ const rocksdb::Slice iterate_lower_bound;
+ const rocksdb::Slice iterate_upper_bound;
+ std::vector<rocksdb::Iterator*> iters;
+public:
+ explicit ShardMergeIteratorImpl(const RocksDBStore* db,
+ const std::string& prefix,
+ const std::vector<rocksdb::ColumnFamilyHandle*>& shards,
+ KeyValueDB::IteratorBounds bounds_)
+ : db(db), keyless(db->comparator), prefix(prefix), bounds(std::move(bounds_)),
+ iterate_lower_bound(make_slice(bounds.lower_bound)),
+ iterate_upper_bound(make_slice(bounds.upper_bound))
+ {
+ iters.reserve(shards.size());
+ auto options = rocksdb::ReadOptions();
+ if (db->cct->_conf->osd_rocksdb_iterator_bounds_enabled) {
+ if (bounds.lower_bound) {
+ options.iterate_lower_bound = &iterate_lower_bound;
+ }
+ if (bounds.upper_bound) {
+ options.iterate_upper_bound = &iterate_upper_bound;
+ }
+ }
+ for (auto& s : shards) {
+ iters.push_back(db->db->NewIterator(options, s));
+ }
+ }
+ ~ShardMergeIteratorImpl() {
+ for (auto& it : iters) {
+ delete it;
+ }
+ }
+ int seek_to_first() override {
+ for (auto& it : iters) {
+ it->SeekToFirst();
+ if (!it->status().ok()) {
+ return -1;
+ }
+ }
+ //all iterators seeked, sort
+ std::sort(iters.begin(), iters.end(), keyless);
+ return 0;
+ }
+ int seek_to_last() override {
+ for (auto& it : iters) {
+ it->SeekToLast();
+ if (!it->status().ok()) {
+ return -1;
+ }
+ }
+ for (size_t i = 1; i < iters.size(); i++) {
+ if (iters[0]->Valid()) {
+ if (iters[i]->Valid()) {
+ if (keyless(iters[0], iters[i])) {
+ std::swap(iters[0], iters[i]);
+ }
+ } else {
+ //iters[i] empty
+ }
+ } else {
+ if (iters[i]->Valid()) {
+ std::swap(iters[0], iters[i]);
+ }
+ }
+ //it might happen that cf was empty
+ if (iters[i]->Valid()) {
+ iters[i]->Next();
+ }
+ }
+ //no need to sort, as at most 1 iterator is valid now
+ return 0;
+ }
+ int upper_bound(const string &after) override {
+ rocksdb::Slice slice_bound(after);
+ for (auto& it : iters) {
+ it->Seek(slice_bound);
+ if (it->Valid() && it->key() == after) {
+ it->Next();
+ }
+ if (!it->status().ok()) {
+ return -1;
+ }
+ }
+ std::sort(iters.begin(), iters.end(), keyless);
+ return 0;
+ }
+ int lower_bound(const string &to) override {
+ rocksdb::Slice slice_bound(to);
+ for (auto& it : iters) {
+ it->Seek(slice_bound);
+ if (!it->status().ok()) {
+ return -1;
+ }
+ }
+ std::sort(iters.begin(), iters.end(), keyless);
+ return 0;
+ }
+ int next() override {
+ int r = -1;
+ if (iters[0]->Valid()) {
+ iters[0]->Next();
+ if (iters[0]->status().ok()) {
+ r = 0;
+ //bubble up
+ for (size_t i = 0; i < iters.size() - 1; i++) {
+ if (keyless(iters[i], iters[i + 1])) {
+ //matches, fixed
+ break;
+ }
+ std::swap(iters[i], iters[i + 1]);
+ }
+ }
+ }
+ return r;
+ }
+ // iters are sorted, so
+ // a[0] < b[0] < c[0] < d[0]
+ // a[0] > a[-1], a[0] > b[-1], a[0] > c[-1], a[0] > d[-1]
+ // so, prev() will be one of:
+ // a[-1], b[-1], c[-1], d[-1]
+ // prev() will be the one that is *largest* of them
+ //
+ // alg:
+ // 1. go prev() on each iterator we can
+ // 2. select largest key from those iterators
+ // 3. go next() on all iterators except (2)
+ // 4. sort
+ int prev() override {
+ std::vector<rocksdb::Iterator*> prev_done;
+ //1
+ for (auto it: iters) {
+ if (it->Valid()) {
+ it->Prev();
+ if (it->Valid()) {
+ prev_done.push_back(it);
+ } else {
+ it->SeekToFirst();
+ }
+ } else {
+ it->SeekToLast();
+ if (it->Valid()) {
+ prev_done.push_back(it);
+ }
+ }
+ }
+ if (prev_done.size() == 0) {
+ /* there is no previous element */
+ if (iters[0]->Valid()) {
+ iters[0]->Prev();
+ ceph_assert(!iters[0]->Valid());
+ }
+ return 0;
+ }
+ //2,3
+ rocksdb::Iterator* highest = prev_done[0];
+ for (size_t i = 1; i < prev_done.size(); i++) {
+ if (keyless(highest, prev_done[i])) {
+ highest->Next();
+ highest = prev_done[i];
+ } else {
+ prev_done[i]->Next();
+ }
+ }
+ //4
+ //insert highest in the beginning, and shift values until we pick highest
+ //untouched rest is sorted - we just prev()/next() them
+ rocksdb::Iterator* hold = highest;
+ for (size_t i = 0; i < iters.size(); i++) {
+ std::swap(hold, iters[i]);
+ if (hold == highest) break;
+ }
+ ceph_assert(hold == highest);
+ return 0;
+ }
+ bool valid() override {
+ return iters[0]->Valid();
+ }
+ string key() override {
+ return iters[0]->key().ToString();
+ }
+ std::pair<std::string, std::string> raw_key() override {
+ return make_pair(prefix, key());
+ }
+ bufferlist value() override {
+ return to_bufferlist(iters[0]->value());
+ }
+ bufferptr value_as_ptr() override {
+ rocksdb::Slice val = iters[0]->value();
+ return bufferptr(val.data(), val.size());
+ }
+ int status() override {
+ return iters[0]->status().ok() ? 0 : -1;
+ }
+};
+
+KeyValueDB::Iterator RocksDBStore::get_iterator(const std::string& prefix, IteratorOpts opts, IteratorBounds bounds)
+{
+ auto cf_it = cf_handles.find(prefix);
+ if (cf_it != cf_handles.end()) {
+ rocksdb::ColumnFamilyHandle* cf = nullptr;
+ if (cf_it->second.handles.size() == 1) {
+ cf = cf_it->second.handles[0];
+ } else if (cct->_conf->osd_rocksdb_iterator_bounds_enabled) {
+ cf = check_cf_handle_bounds(cf_it, bounds);
+ }
+ if (cf) {
+ return std::make_shared<CFIteratorImpl>(
+ this,
+ prefix,
+ cf,
+ std::move(bounds));
+ } else {
+ return std::make_shared<ShardMergeIteratorImpl>(
+ this,
+ prefix,
+ cf_it->second.handles,
+ std::move(bounds));
+ }
+ } else {
+ // use wholespace engine if no cfs are configured
+ // or use default cf otherwise as there is no
+ // matching cf for the specified prefix.
+ auto w_it = cf_handles.size() == 0 || prefix.empty() ?
+ get_wholespace_iterator(opts) :
+ get_default_cf_iterator();
+ return KeyValueDB::make_iterator(prefix, w_it);
+ }
+}
+
+RocksDBStore::WholeSpaceIterator RocksDBStore::new_shard_iterator(rocksdb::ColumnFamilyHandle* cf)
+{
+ return std::make_shared<RocksDBWholeSpaceIteratorImpl>(
+ this,
+ cf,
+ 0);
+}
+
+KeyValueDB::Iterator RocksDBStore::new_shard_iterator(rocksdb::ColumnFamilyHandle* cf,
+ const std::string& prefix,
+ IteratorBounds bounds)
+{
+ return std::make_shared<CFIteratorImpl>(
+ this,
+ prefix,
+ cf,
+ std::move(bounds));
+}
+
+RocksDBStore::WholeSpaceIterator RocksDBStore::get_wholespace_iterator(IteratorOpts opts)
+{
+ if (cf_handles.size() == 0) {
+ return std::make_shared<RocksDBWholeSpaceIteratorImpl>(
+ this, default_cf, opts);
+ } else {
+ return std::make_shared<WholeMergeIteratorImpl>(this);
+ }
+}
+
+RocksDBStore::WholeSpaceIterator RocksDBStore::get_default_cf_iterator()
+{
+ return std::make_shared<RocksDBWholeSpaceIteratorImpl>(this, default_cf, 0);
+}
+
+int RocksDBStore::prepare_for_reshard(const std::string& new_sharding,
+ RocksDBStore::columns_t& to_process_columns)
+{
+ //0. lock db from opening
+ //1. list existing columns
+ //2. apply merge operator to (main + columns) opts
+ //3. prepare std::vector<rocksdb::ColumnFamilyDescriptor> existing_cfs
+ //4. open db, acquire existing column handles
+ //5. calculate missing columns
+ //6. create missing columns
+ //7. construct cf_handles according to new sharding
+ //8. check is all cf_handles are filled
+
+ bool b;
+ std::vector<ColumnFamily> new_sharding_def;
+ char const* error_position;
+ std::string error_msg;
+ b = parse_sharding_def(new_sharding, new_sharding_def, &error_position, &error_msg);
+ if (!b) {
+ dout(1) << __func__ << " bad sharding: " << dendl;
+ dout(1) << __func__ << new_sharding << dendl;
+ dout(1) << __func__ << std::string(error_position - &new_sharding[0], ' ') << "^" << error_msg << dendl;
+ return -EINVAL;
+ }
+
+ //0. lock db from opening
+ std::string stored_sharding_text;
+ rocksdb::ReadFileToString(env,
+ sharding_def_file,
+ &stored_sharding_text);
+ if (stored_sharding_text.find(resharding_column_lock) == string::npos) {
+ rocksdb::Status status;
+ if (stored_sharding_text.size() != 0)
+ stored_sharding_text += " ";
+ stored_sharding_text += resharding_column_lock;
+ env->CreateDir(sharding_def_dir);
+ status = rocksdb::WriteStringToFile(env, stored_sharding_text,
+ sharding_def_file, true);
+ if (!status.ok()) {
+ derr << __func__ << " cannot write to " << sharding_def_file << dendl;
+ return -EIO;
+ }
+ }
+
+ //1. list existing columns
+
+ rocksdb::Status status;
+ std::vector<std::string> existing_columns;
+ rocksdb::Options opt;
+ int r = load_rocksdb_options(false, opt);
+ if (r) {
+ dout(1) << __func__ << " load rocksdb options failed" << dendl;
+ return r;
+ }
+ status = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(opt), path, &existing_columns);
+ if (!status.ok()) {
+ derr << "Unable to list column families: " << status.ToString() << dendl;
+ return -EINVAL;
+ }
+ dout(5) << "existing columns = " << existing_columns << dendl;
+
+ //2. apply merge operator to (main + columns) opts
+ //3. prepare std::vector<rocksdb::ColumnFamilyDescriptor> cfs_to_open
+
+ std::vector<rocksdb::ColumnFamilyDescriptor> cfs_to_open;
+ for (const auto& full_name : existing_columns) {
+ //split col_name to <prefix>-<number>
+ std::string base_name;
+ size_t pos = full_name.find('-');
+ if (std::string::npos == pos)
+ base_name = full_name;
+ else
+ base_name = full_name.substr(0,pos);
+
+ rocksdb::ColumnFamilyOptions cf_opt(opt);
+ // search if we have options for this column
+ std::string options;
+ for (const auto& nsd : new_sharding_def) {
+ if (nsd.name == base_name) {
+ options = nsd.options;
+ break;
+ }
+ }
+ int r = update_column_family_options(base_name, options, &cf_opt);
+ if (r != 0) {
+ return r;
+ }
+ cfs_to_open.emplace_back(full_name, cf_opt);
+ }
+
+ //4. open db, acquire existing column handles
+ std::vector<rocksdb::ColumnFamilyHandle*> handles;
+ status = rocksdb::DB::Open(rocksdb::DBOptions(opt),
+ path, cfs_to_open, &handles, &db);
+ if (!status.ok()) {
+ derr << status.ToString() << dendl;
+ return -EINVAL;
+ }
+ for (size_t i = 0; i < cfs_to_open.size(); i++) {
+ dout(10) << "column " << cfs_to_open[i].name << " handle " << (void*)handles[i] << dendl;
+ }
+
+ //5. calculate missing columns
+ std::vector<std::string> new_sharding_columns;
+ std::vector<std::string> missing_columns;
+ sharding_def_to_columns(new_sharding_def,
+ new_sharding_columns);
+ dout(5) << "target columns = " << new_sharding_columns << dendl;
+ for (const auto& n : new_sharding_columns) {
+ bool found = false;
+ for (const auto& e : existing_columns) {
+ if (n == e) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ missing_columns.push_back(n);
+ }
+ }
+ dout(5) << "missing columns = " << missing_columns << dendl;
+
+ //6. create missing columns
+ for (const auto& full_name : missing_columns) {
+ std::string base_name;
+ size_t pos = full_name.find('-');
+ if (std::string::npos == pos)
+ base_name = full_name;
+ else
+ base_name = full_name.substr(0,pos);
+
+ rocksdb::ColumnFamilyOptions cf_opt(opt);
+ // search if we have options for this column
+ std::string options;
+ for (const auto& nsd : new_sharding_def) {
+ if (nsd.name == base_name) {
+ options = nsd.options;
+ break;
+ }
+ }
+ int r = update_column_family_options(base_name, options, &cf_opt);
+ if (r != 0) {
+ return r;
+ }
+ rocksdb::ColumnFamilyHandle *cf;
+ status = db->CreateColumnFamily(cf_opt, full_name, &cf);
+ if (!status.ok()) {
+ derr << __func__ << " Failed to create rocksdb column family: "
+ << full_name << dendl;
+ return -EINVAL;
+ }
+ dout(10) << "created column " << full_name << " handle = " << (void*)cf << dendl;
+ existing_columns.push_back(full_name);
+ handles.push_back(cf);
+ }
+
+ //7. construct cf_handles according to new sharding
+ for (size_t i = 0; i < existing_columns.size(); i++) {
+ std::string full_name = existing_columns[i];
+ rocksdb::ColumnFamilyHandle *cf = handles[i];
+ std::string base_name;
+ size_t shard_idx = 0;
+ size_t pos = full_name.find('-');
+ dout(10) << "processing column " << full_name << dendl;
+ if (std::string::npos == pos) {
+ base_name = full_name;
+ } else {
+ base_name = full_name.substr(0,pos);
+ shard_idx = atoi(full_name.substr(pos+1).c_str());
+ }
+ if (rocksdb::kDefaultColumnFamilyName == base_name) {
+ default_cf = handles[i];
+ must_close_default_cf = true;
+ std::unique_ptr<rocksdb::ColumnFamilyHandle, cf_deleter_t> ptr{
+ cf, [](rocksdb::ColumnFamilyHandle*) {}};
+ to_process_columns.emplace(full_name, std::move(ptr));
+ } else {
+ for (const auto& nsd : new_sharding_def) {
+ if (nsd.name == base_name) {
+ if (shard_idx < nsd.shard_cnt) {
+ add_column_family(base_name, nsd.hash_l, nsd.hash_h, shard_idx, cf);
+ } else {
+ //ignore columns with index larger then shard count
+ }
+ break;
+ }
+ }
+ std::unique_ptr<rocksdb::ColumnFamilyHandle, cf_deleter_t> ptr{
+ cf, [this](rocksdb::ColumnFamilyHandle* handle) {
+ db->DestroyColumnFamilyHandle(handle);
+ }};
+ to_process_columns.emplace(full_name, std::move(ptr));
+ }
+ }
+
+ //8. check if all cf_handles are filled
+ for (const auto& col : cf_handles) {
+ for (size_t i = 0; i < col.second.handles.size(); i++) {
+ if (col.second.handles[i] == nullptr) {
+ derr << "missing handle for column " << col.first << " shard " << i << dendl;
+ return -EIO;
+ }
+ }
+ }
+ return 0;
+}
+
+int RocksDBStore::reshard_cleanup(const RocksDBStore::columns_t& current_columns)
+{
+ std::vector<std::string> new_sharding_columns;
+ for (const auto& [name, handle] : cf_handles) {
+ if (handle.handles.size() == 1) {
+ new_sharding_columns.push_back(name);
+ } else {
+ for (size_t i = 0; i < handle.handles.size(); i++) {
+ new_sharding_columns.push_back(name + "-" + std::to_string(i));
+ }
+ }
+ }
+
+ for (auto& [name, handle] : current_columns) {
+ auto found = std::find(new_sharding_columns.begin(),
+ new_sharding_columns.end(),
+ name) != new_sharding_columns.end();
+ if (found || name == rocksdb::kDefaultColumnFamilyName) {
+ dout(5) << "Column " << name << " is part of new sharding." << dendl;
+ continue;
+ }
+ dout(5) << "Column " << name << " not part of new sharding. Deleting." << dendl;
+
+ // verify that column is empty
+ std::unique_ptr<rocksdb::Iterator> it{
+ db->NewIterator(rocksdb::ReadOptions(), handle.get())};
+ ceph_assert(it);
+ it->SeekToFirst();
+ ceph_assert(!it->Valid());
+
+ if (rocksdb::Status status = db->DropColumnFamily(handle.get()); !status.ok()) {
+ derr << __func__ << " Failed to drop column: " << name << dendl;
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+
+int RocksDBStore::reshard(const std::string& new_sharding, const RocksDBStore::resharding_ctrl* ctrl_in)
+{
+
+ resharding_ctrl ctrl = ctrl_in ? *ctrl_in : resharding_ctrl();
+ size_t bytes_in_batch = 0;
+ size_t keys_in_batch = 0;
+ size_t bytes_per_iterator = 0;
+ size_t keys_per_iterator = 0;
+ size_t keys_processed = 0;
+ size_t keys_moved = 0;
+
+ auto flush_batch = [&](rocksdb::WriteBatch* batch) {
+ dout(10) << "flushing batch, " << keys_in_batch << " keys, for "
+ << bytes_in_batch << " bytes" << dendl;
+ rocksdb::WriteOptions woptions;
+ woptions.sync = true;
+ rocksdb::Status s = db->Write(woptions, batch);
+ ceph_assert(s.ok());
+ bytes_in_batch = 0;
+ keys_in_batch = 0;
+ batch->Clear();
+ };
+
+ auto process_column = [&](rocksdb::ColumnFamilyHandle* handle,
+ const std::string& fixed_prefix)
+ {
+ dout(5) << " column=" << (void*)handle << " prefix=" << fixed_prefix << dendl;
+ std::unique_ptr<rocksdb::Iterator> it{
+ db->NewIterator(rocksdb::ReadOptions(), handle)};
+ ceph_assert(it);
+
+ rocksdb::WriteBatch bat;
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ rocksdb::Slice raw_key = it->key();
+ dout(30) << "key=" << pretty_binary_string(raw_key.ToString()) << dendl;
+ //check if need to refresh iterator
+ if (bytes_per_iterator >= ctrl.bytes_per_iterator ||
+ keys_per_iterator >= ctrl.keys_per_iterator) {
+ dout(8) << "refreshing iterator" << dendl;
+ bytes_per_iterator = 0;
+ keys_per_iterator = 0;
+ std::string raw_key_str = raw_key.ToString();
+ it.reset(db->NewIterator(rocksdb::ReadOptions(), handle));
+ ceph_assert(it);
+ it->Seek(raw_key_str);
+ ceph_assert(it->Valid());
+ raw_key = it->key();
+ }
+ rocksdb::Slice value = it->value();
+ std::string prefix, key;
+ if (fixed_prefix.size() == 0) {
+ split_key(raw_key, &prefix, &key);
+ } else {
+ prefix = fixed_prefix;
+ key = raw_key.ToString();
+ }
+ keys_processed++;
+ if ((keys_processed % 10000) == 0) {
+ dout(10) << "processed " << keys_processed << " keys, moved " << keys_moved << dendl;
+ }
+ rocksdb::ColumnFamilyHandle* new_handle = get_cf_handle(prefix, key);
+ if (new_handle == nullptr) {
+ new_handle = default_cf;
+ }
+ if (handle == new_handle) {
+ continue;
+ }
+ std::string new_raw_key;
+ if (new_handle == default_cf) {
+ new_raw_key = combine_strings(prefix, key);
+ } else {
+ new_raw_key = key;
+ }
+ bat.Delete(handle, raw_key);
+ bat.Put(new_handle, new_raw_key, value);
+ dout(25) << "moving " << (void*)handle << "/" << pretty_binary_string(raw_key.ToString()) <<
+ " to " << (void*)new_handle << "/" << pretty_binary_string(new_raw_key) <<
+ " size " << value.size() << dendl;
+ keys_moved++;
+ bytes_in_batch += new_raw_key.size() * 2 + value.size();
+ keys_in_batch++;
+ bytes_per_iterator += new_raw_key.size() * 2 + value.size();
+ keys_per_iterator++;
+
+ //check if need to write batch
+ if (bytes_in_batch >= ctrl.bytes_per_batch ||
+ keys_in_batch >= ctrl.keys_per_batch) {
+ flush_batch(&bat);
+ if (ctrl.unittest_fail_after_first_batch) {
+ return -1000;
+ }
+ }
+ }
+ if (bat.Count() > 0) {
+ flush_batch(&bat);
+ }
+ return 0;
+ };
+
+ auto close_column_handles = make_scope_guard([this] {
+ cf_handles.clear();
+ close();
+ });
+ columns_t to_process_columns;
+ int r = prepare_for_reshard(new_sharding, to_process_columns);
+ if (r != 0) {
+ dout(1) << "failed to prepare db for reshard" << dendl;
+ return r;
+ }
+
+ for (auto& [name, handle] : to_process_columns) {
+ dout(5) << "Processing column=" << name
+ << " handle=" << handle.get() << dendl;
+ if (name == rocksdb::kDefaultColumnFamilyName) {
+ ceph_assert(handle.get() == default_cf);
+ r = process_column(default_cf, std::string());
+ } else {
+ std::string fixed_prefix = name.substr(0, name.find('-'));
+ dout(10) << "Prefix: " << fixed_prefix << dendl;
+ r = process_column(handle.get(), fixed_prefix);
+ }
+ if (r != 0) {
+ derr << "Error processing column " << name << dendl;
+ return r;
+ }
+ if (ctrl.unittest_fail_after_processing_column) {
+ return -1001;
+ }
+ }
+
+ r = reshard_cleanup(to_process_columns);
+ if (r != 0) {
+ dout(5) << "failed to cleanup after reshard" << dendl;
+ return r;
+ }
+
+ if (ctrl.unittest_fail_after_successful_processing) {
+ return -1002;
+ }
+ env->CreateDir(sharding_def_dir);
+ if (auto status = rocksdb::WriteStringToFile(env, new_sharding,
+ sharding_def_file, true);
+ !status.ok()) {
+ derr << __func__ << " cannot write to " << sharding_def_file << dendl;
+ return -EIO;
+ }
+
+ return r;
+}
+
+bool RocksDBStore::get_sharding(std::string& sharding) {
+ rocksdb::Status status;
+ std::string stored_sharding_text;
+ bool result = false;
+ sharding.clear();
+
+ status = env->FileExists(sharding_def_file);
+ if (status.ok()) {
+ status = rocksdb::ReadFileToString(env,
+ sharding_def_file,
+ &stored_sharding_text);
+ if(status.ok()) {
+ result = true;
+ sharding = stored_sharding_text;
+ }
+ }
+ return result;
+}
diff --git a/src/kv/RocksDBStore.h b/src/kv/RocksDBStore.h
new file mode 100644
index 000000000..01c84f2d0
--- /dev/null
+++ b/src/kv/RocksDBStore.h
@@ -0,0 +1,551 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+#ifndef ROCKS_DB_STORE_H
+#define ROCKS_DB_STORE_H
+
+#include "include/types.h"
+#include "include/buffer_fwd.h"
+#include "KeyValueDB.h"
+#include <set>
+#include <map>
+#include <string>
+#include <memory>
+#include <boost/scoped_ptr.hpp>
+#include "rocksdb/write_batch.h"
+#include "rocksdb/perf_context.h"
+#include "rocksdb/iostats_context.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/table.h"
+#include "rocksdb/db.h"
+#include "kv/rocksdb_cache/BinnedLRUCache.h"
+#include <errno.h>
+#include "common/errno.h"
+#include "common/dout.h"
+#include "include/ceph_assert.h"
+#include "include/common_fwd.h"
+#include "common/Formatter.h"
+#include "common/Cond.h"
+#include "common/ceph_context.h"
+#include "common/PriorityCache.h"
+#include "common/pretty_binary.h"
+
+enum {
+ l_rocksdb_first = 34300,
+ l_rocksdb_get_latency,
+ l_rocksdb_submit_latency,
+ l_rocksdb_submit_sync_latency,
+ l_rocksdb_compact,
+ l_rocksdb_compact_range,
+ l_rocksdb_compact_queue_merge,
+ l_rocksdb_compact_queue_len,
+ l_rocksdb_write_wal_time,
+ l_rocksdb_write_memtable_time,
+ l_rocksdb_write_delay_time,
+ l_rocksdb_write_pre_and_post_process_time,
+ l_rocksdb_last,
+};
+
+namespace rocksdb{
+ class DB;
+ class Env;
+ class Cache;
+ class FilterPolicy;
+ class Snapshot;
+ class Slice;
+ class WriteBatch;
+ class Iterator;
+ class Logger;
+ class ColumnFamilyHandle;
+ struct Options;
+ struct BlockBasedTableOptions;
+ struct DBOptions;
+ struct ColumnFamilyOptions;
+}
+
+extern rocksdb::Logger *create_rocksdb_ceph_logger();
+
+inline rocksdb::Slice make_slice(const std::optional<std::string>& bound) {
+ if (bound) {
+ return {*bound};
+ } else {
+ return {};
+ }
+}
+
+/**
+ * Uses RocksDB to implement the KeyValueDB interface
+ */
+class RocksDBStore : public KeyValueDB {
+ CephContext *cct;
+ PerfCounters *logger;
+ std::string path;
+ std::map<std::string,std::string> kv_options;
+ void *priv;
+ rocksdb::DB *db;
+ rocksdb::Env *env;
+ const rocksdb::Comparator* comparator;
+ std::shared_ptr<rocksdb::Statistics> dbstats;
+ rocksdb::BlockBasedTableOptions bbt_opts;
+ std::string options_str;
+
+ uint64_t cache_size = 0;
+ bool set_cache_flag = false;
+ friend class ShardMergeIteratorImpl;
+ friend class CFIteratorImpl;
+ friend class WholeMergeIteratorImpl;
+ /*
+ * See RocksDB's definition of a column family(CF) and how to use it.
+ * The interfaces of KeyValueDB is extended, when a column family is created.
+ * Prefix will be the name of column family to use.
+ */
+public:
+ struct ColumnFamily {
+ std::string name; //< name of this individual column family
+ size_t shard_cnt; //< count of shards
+ std::string options; //< configure option string for this CF
+ uint32_t hash_l; //< first character to take for hash calc.
+ uint32_t hash_h; //< last character to take for hash calc.
+ ColumnFamily(const std::string &name, size_t shard_cnt, const std::string &options,
+ uint32_t hash_l, uint32_t hash_h)
+ : name(name), shard_cnt(shard_cnt), options(options), hash_l(hash_l), hash_h(hash_h) {}
+ };
+private:
+ friend std::ostream& operator<<(std::ostream& out, const ColumnFamily& cf);
+
+ bool must_close_default_cf = false;
+ rocksdb::ColumnFamilyHandle *default_cf = nullptr;
+
+ /// column families in use, name->handles
+ struct prefix_shards {
+ uint32_t hash_l; //< first character to take for hash calc.
+ uint32_t hash_h; //< last character to take for hash calc.
+ std::vector<rocksdb::ColumnFamilyHandle *> handles;
+ };
+ std::unordered_map<std::string, prefix_shards> cf_handles;
+ typedef decltype(cf_handles)::iterator cf_handles_iterator;
+ std::unordered_map<uint32_t, std::string> cf_ids_to_prefix;
+ std::unordered_map<std::string, rocksdb::BlockBasedTableOptions> cf_bbt_opts;
+
+ void add_column_family(const std::string& cf_name, uint32_t hash_l, uint32_t hash_h,
+ size_t shard_idx, rocksdb::ColumnFamilyHandle *handle);
+ bool is_column_family(const std::string& prefix);
+ std::string_view get_key_hash_view(const prefix_shards& shards, const char* key, const size_t keylen);
+ rocksdb::ColumnFamilyHandle *get_key_cf(const prefix_shards& shards, const char* key, const size_t keylen);
+ rocksdb::ColumnFamilyHandle *get_cf_handle(const std::string& prefix, const std::string& key);
+ rocksdb::ColumnFamilyHandle *get_cf_handle(const std::string& prefix, const char* key, size_t keylen);
+ rocksdb::ColumnFamilyHandle *check_cf_handle_bounds(const cf_handles_iterator& it, const IteratorBounds& bounds);
+
+ int submit_common(rocksdb::WriteOptions& woptions, KeyValueDB::Transaction t);
+ int install_cf_mergeop(const std::string &cf_name, rocksdb::ColumnFamilyOptions *cf_opt);
+ int create_db_dir();
+ int do_open(std::ostream &out, bool create_if_missing, bool open_readonly,
+ const std::string& cfs="");
+ int load_rocksdb_options(bool create_if_missing, rocksdb::Options& opt);
+public:
+ static bool parse_sharding_def(const std::string_view text_def,
+ std::vector<ColumnFamily>& sharding_def,
+ char const* *error_position = nullptr,
+ std::string *error_msg = nullptr);
+ const rocksdb::Comparator* get_comparator() const {
+ return comparator;
+ }
+
+private:
+ static void sharding_def_to_columns(const std::vector<ColumnFamily>& sharding_def,
+ std::vector<std::string>& columns);
+ int create_shards(const rocksdb::Options& opt,
+ const std::vector<ColumnFamily>& sharding_def);
+ int apply_sharding(const rocksdb::Options& opt,
+ const std::string& sharding_text);
+ int verify_sharding(const rocksdb::Options& opt,
+ std::vector<rocksdb::ColumnFamilyDescriptor>& existing_cfs,
+ std::vector<std::pair<size_t, RocksDBStore::ColumnFamily> >& existing_cfs_shard,
+ std::vector<rocksdb::ColumnFamilyDescriptor>& missing_cfs,
+ std::vector<std::pair<size_t, RocksDBStore::ColumnFamily> >& missing_cfs_shard);
+ std::shared_ptr<rocksdb::Cache> create_block_cache(const std::string& cache_type, size_t cache_size, double cache_prio_high = 0.0);
+ int split_column_family_options(const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* column_opts_map,
+ std::string* block_cache_opt);
+ int apply_block_cache_options(const std::string& column_name,
+ const std::string& block_cache_opt,
+ rocksdb::ColumnFamilyOptions* cf_opt);
+ int update_column_family_options(const std::string& base_name,
+ const std::string& more_options,
+ rocksdb::ColumnFamilyOptions* cf_opt);
+ // manage async compactions
+ ceph::mutex compact_queue_lock =
+ ceph::make_mutex("RocksDBStore::compact_thread_lock");
+ ceph::condition_variable compact_queue_cond;
+ std::list<std::pair<std::string,std::string>> compact_queue;
+ bool compact_queue_stop;
+ class CompactThread : public Thread {
+ RocksDBStore *db;
+ public:
+ explicit CompactThread(RocksDBStore *d) : db(d) {}
+ void *entry() override {
+ db->compact_thread_entry();
+ return NULL;
+ }
+ friend class RocksDBStore;
+ } compact_thread;
+
+ void compact_thread_entry();
+
+ void compact_range(const std::string& start, const std::string& end);
+ void compact_range_async(const std::string& start, const std::string& end);
+ int tryInterpret(const std::string& key, const std::string& val,
+ rocksdb::Options& opt);
+
+public:
+ /// compact the underlying rocksdb store
+ bool compact_on_mount;
+ bool disableWAL;
+ uint64_t get_delete_range_threshold() const {
+ return cct->_conf.get_val<uint64_t>("rocksdb_delete_range_threshold");
+ }
+
+ void compact() override;
+
+ void compact_async() override {
+ compact_range_async({}, {});
+ }
+
+ int ParseOptionsFromString(const std::string& opt_str, rocksdb::Options& opt);
+ static int ParseOptionsFromStringStatic(
+ CephContext* cct,
+ const std::string& opt_str,
+ rocksdb::Options &opt,
+ std::function<int(const std::string&, const std::string&, rocksdb::Options&)> interp);
+ static int _test_init(const std::string& dir);
+ int init(std::string options_str) override;
+ /// compact rocksdb for all keys with a given prefix
+ void compact_prefix(const std::string& prefix) override {
+ compact_range(prefix, past_prefix(prefix));
+ }
+ void compact_prefix_async(const std::string& prefix) override {
+ compact_range_async(prefix, past_prefix(prefix));
+ }
+
+ void compact_range(const std::string& prefix, const std::string& start,
+ const std::string& end) override {
+ compact_range(combine_strings(prefix, start), combine_strings(prefix, end));
+ }
+ void compact_range_async(const std::string& prefix, const std::string& start,
+ const std::string& end) override {
+ compact_range_async(combine_strings(prefix, start), combine_strings(prefix, end));
+ }
+
+ RocksDBStore(CephContext *c, const std::string &path, std::map<std::string,std::string> opt, void *p) :
+ cct(c),
+ logger(NULL),
+ path(path),
+ kv_options(opt),
+ priv(p),
+ db(NULL),
+ env(static_cast<rocksdb::Env*>(p)),
+ comparator(nullptr),
+ dbstats(NULL),
+ compact_queue_stop(false),
+ compact_thread(this),
+ compact_on_mount(false),
+ disableWAL(false)
+ {}
+
+ ~RocksDBStore() override;
+
+ static bool check_omap_dir(std::string &omap_dir);
+ /// Opens underlying db
+ int open(std::ostream &out, const std::string& cfs="") override {
+ return do_open(out, false, false, cfs);
+ }
+ /// Creates underlying db if missing and opens it
+ int create_and_open(std::ostream &out,
+ const std::string& cfs="") override;
+
+ int open_read_only(std::ostream &out, const std::string& cfs="") override {
+ return do_open(out, false, true, cfs);
+ }
+
+ void close() override;
+
+ int repair(std::ostream &out) override;
+ void split_stats(const std::string &s, char delim, std::vector<std::string> &elems);
+ void get_statistics(ceph::Formatter *f) override;
+
+ PerfCounters *get_perf_counters() override
+ {
+ return logger;
+ }
+
+ bool get_property(
+ const std::string &property,
+ uint64_t *out) final;
+
+ int64_t estimate_prefix_size(const std::string& prefix,
+ const std::string& key_prefix) override;
+ struct RocksWBHandler;
+ class RocksDBTransactionImpl : public KeyValueDB::TransactionImpl {
+ public:
+ rocksdb::WriteBatch bat;
+ RocksDBStore *db;
+
+ explicit RocksDBTransactionImpl(RocksDBStore *_db);
+ private:
+ void put_bat(
+ rocksdb::WriteBatch& bat,
+ rocksdb::ColumnFamilyHandle *cf,
+ const std::string &k,
+ const ceph::bufferlist &to_set_bl);
+ public:
+ void set(
+ const std::string &prefix,
+ const std::string &k,
+ const ceph::bufferlist &bl) override;
+ void set(
+ const std::string &prefix,
+ const char *k,
+ size_t keylen,
+ const ceph::bufferlist &bl) override;
+ void rmkey(
+ const std::string &prefix,
+ const std::string &k) override;
+ void rmkey(
+ const std::string &prefix,
+ const char *k,
+ size_t keylen) override;
+ void rm_single_key(
+ const std::string &prefix,
+ const std::string &k) override;
+ void rmkeys_by_prefix(
+ const std::string &prefix
+ ) override;
+ void rm_range_keys(
+ const std::string &prefix,
+ const std::string &start,
+ const std::string &end) override;
+ void merge(
+ const std::string& prefix,
+ const std::string& k,
+ const ceph::bufferlist &bl) override;
+ };
+
+ KeyValueDB::Transaction get_transaction() override {
+ return std::make_shared<RocksDBTransactionImpl>(this);
+ }
+
+ int submit_transaction(KeyValueDB::Transaction t) override;
+ int submit_transaction_sync(KeyValueDB::Transaction t) override;
+ int get(
+ const std::string &prefix,
+ const std::set<std::string> &key,
+ std::map<std::string, ceph::bufferlist> *out
+ ) override;
+ int get(
+ const std::string &prefix,
+ const std::string &key,
+ ceph::bufferlist *out
+ ) override;
+ int get(
+ const std::string &prefix,
+ const char *key,
+ size_t keylen,
+ ceph::bufferlist *out) override;
+
+
+ class RocksDBWholeSpaceIteratorImpl :
+ public KeyValueDB::WholeSpaceIteratorImpl {
+ protected:
+ rocksdb::Iterator *dbiter;
+ public:
+ explicit RocksDBWholeSpaceIteratorImpl(const RocksDBStore* db,
+ rocksdb::ColumnFamilyHandle* cf,
+ const KeyValueDB::IteratorOpts opts)
+ {
+ rocksdb::ReadOptions options = rocksdb::ReadOptions();
+ if (opts & ITERATOR_NOCACHE)
+ options.fill_cache=false;
+ dbiter = db->db->NewIterator(options, cf);
+ }
+ ~RocksDBWholeSpaceIteratorImpl() override;
+
+ int seek_to_first() override;
+ int seek_to_first(const std::string &prefix) override;
+ int seek_to_last() override;
+ int seek_to_last(const std::string &prefix) override;
+ int upper_bound(const std::string &prefix, const std::string &after) override;
+ int lower_bound(const std::string &prefix, const std::string &to) override;
+ bool valid() override;
+ int next() override;
+ int prev() override;
+ std::string key() override;
+ std::pair<std::string,std::string> raw_key() override;
+ bool raw_key_is_prefixed(const std::string &prefix) override;
+ ceph::bufferlist value() override;
+ ceph::bufferptr value_as_ptr() override;
+ int status() override;
+ size_t key_size() override;
+ size_t value_size() override;
+ };
+
+ Iterator get_iterator(const std::string& prefix, IteratorOpts opts = 0, IteratorBounds = IteratorBounds()) override;
+private:
+ /// this iterator spans single cf
+ WholeSpaceIterator new_shard_iterator(rocksdb::ColumnFamilyHandle* cf);
+ Iterator new_shard_iterator(rocksdb::ColumnFamilyHandle* cf,
+ const std::string& prefix, IteratorBounds bound);
+public:
+ /// Utility
+ static std::string combine_strings(const std::string &prefix, const std::string &value) {
+ std::string out = prefix;
+ out.push_back(0);
+ out.append(value);
+ return out;
+ }
+ static void combine_strings(const std::string &prefix,
+ const char *key, size_t keylen,
+ std::string *out) {
+ out->reserve(prefix.size() + 1 + keylen);
+ *out = prefix;
+ out->push_back(0);
+ out->append(key, keylen);
+ }
+
+ static int split_key(rocksdb::Slice in, std::string *prefix, std::string *key);
+
+ static std::string past_prefix(const std::string &prefix);
+
+ class MergeOperatorRouter;
+ class MergeOperatorLinker;
+ friend class MergeOperatorRouter;
+ int set_merge_operator(
+ const std::string& prefix,
+ std::shared_ptr<KeyValueDB::MergeOperator> mop) override;
+ std::string assoc_name; ///< Name of associative operator
+
+ uint64_t get_estimated_size(std::map<std::string,uint64_t> &extra) override {
+ DIR *store_dir = opendir(path.c_str());
+ if (!store_dir) {
+ lderr(cct) << __func__ << " something happened opening the store: "
+ << cpp_strerror(errno) << dendl;
+ return 0;
+ }
+
+ uint64_t total_size = 0;
+ uint64_t sst_size = 0;
+ uint64_t log_size = 0;
+ uint64_t misc_size = 0;
+
+ struct dirent *entry = NULL;
+ while ((entry = readdir(store_dir)) != NULL) {
+ std::string n(entry->d_name);
+
+ if (n == "." || n == "..")
+ continue;
+
+ std::string fpath = path + '/' + n;
+ struct stat s;
+ int err = stat(fpath.c_str(), &s);
+ if (err < 0)
+ err = -errno;
+ // we may race against rocksdb while reading files; this should only
+ // happen when those files are being updated, data is being shuffled
+ // and files get removed, in which case there's not much of a problem
+ // as we'll get to them next time around.
+ if (err == -ENOENT) {
+ continue;
+ }
+ if (err < 0) {
+ lderr(cct) << __func__ << " error obtaining stats for " << fpath
+ << ": " << cpp_strerror(err) << dendl;
+ goto err;
+ }
+
+ size_t pos = n.find_last_of('.');
+ if (pos == std::string::npos) {
+ misc_size += s.st_size;
+ continue;
+ }
+
+ std::string ext = n.substr(pos+1);
+ if (ext == "sst") {
+ sst_size += s.st_size;
+ } else if (ext == "log") {
+ log_size += s.st_size;
+ } else {
+ misc_size += s.st_size;
+ }
+ }
+
+ total_size = sst_size + log_size + misc_size;
+
+ extra["sst"] = sst_size;
+ extra["log"] = log_size;
+ extra["misc"] = misc_size;
+ extra["total"] = total_size;
+
+err:
+ closedir(store_dir);
+ return total_size;
+ }
+
+ virtual int64_t get_cache_usage() const override {
+ return static_cast<int64_t>(bbt_opts.block_cache->GetUsage());
+ }
+
+ virtual int64_t get_cache_usage(std::string prefix) const override {
+ auto it = cf_bbt_opts.find(prefix);
+ if (it != cf_bbt_opts.end() && it->second.block_cache) {
+ return static_cast<int64_t>(it->second.block_cache->GetUsage());
+ }
+ return -EINVAL;
+ }
+
+ int set_cache_size(uint64_t s) override {
+ cache_size = s;
+ set_cache_flag = true;
+ return 0;
+ }
+
+ virtual std::shared_ptr<PriorityCache::PriCache>
+ get_priority_cache() const override {
+ return std::dynamic_pointer_cast<PriorityCache::PriCache>(
+ bbt_opts.block_cache);
+ }
+
+ virtual std::shared_ptr<PriorityCache::PriCache>
+ get_priority_cache(std::string prefix) const override {
+ auto it = cf_bbt_opts.find(prefix);
+ if (it != cf_bbt_opts.end()) {
+ return std::dynamic_pointer_cast<PriorityCache::PriCache>(
+ it->second.block_cache);
+ }
+ return nullptr;
+ }
+
+ WholeSpaceIterator get_wholespace_iterator(IteratorOpts opts = 0) override;
+private:
+ WholeSpaceIterator get_default_cf_iterator();
+
+ using cf_deleter_t = std::function<void(rocksdb::ColumnFamilyHandle*)>;
+ using columns_t = std::map<std::string,
+ std::unique_ptr<rocksdb::ColumnFamilyHandle,
+ cf_deleter_t>>;
+ int prepare_for_reshard(const std::string& new_sharding,
+ columns_t& to_process_columns);
+ int reshard_cleanup(const columns_t& current_columns);
+public:
+ struct resharding_ctrl {
+ size_t bytes_per_iterator = 10000000; /// amount of data to process before refreshing iterator
+ size_t keys_per_iterator = 10000;
+ size_t bytes_per_batch = 1000000; /// amount of data before submitting batch
+ size_t keys_per_batch = 1000;
+ bool unittest_fail_after_first_batch = false;
+ bool unittest_fail_after_processing_column = false;
+ bool unittest_fail_after_successful_processing = false;
+ };
+ int reshard(const std::string& new_sharding, const resharding_ctrl* ctrl = nullptr);
+ bool get_sharding(std::string& sharding);
+
+};
+
+#endif
diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.cc b/src/kv/rocksdb_cache/BinnedLRUCache.cc
new file mode 100644
index 000000000..2e66c16ee
--- /dev/null
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.cc
@@ -0,0 +1,719 @@
+// Copyright (c) 2018-Present Red Hat Inc. All rights reserved.
+//
+// Copyright (c) 2011-2018, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 and Apache 2.0 License
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "BinnedLRUCache.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+
+#define dout_context cct
+#define dout_subsys ceph_subsys_rocksdb
+#undef dout_prefix
+#define dout_prefix *_dout << "rocksdb: "
+
+namespace rocksdb_cache {
+
+BinnedLRUHandleTable::BinnedLRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+ Resize();
+}
+
+BinnedLRUHandleTable::~BinnedLRUHandleTable() {
+ ApplyToAllCacheEntries([](BinnedLRUHandle* h) {
+ if (h->refs == 1) {
+ h->Free();
+ }
+ });
+ delete[] list_;
+}
+
+BinnedLRUHandle* BinnedLRUHandleTable::Lookup(const rocksdb::Slice& key, uint32_t hash) {
+ return *FindPointer(key, hash);
+}
+
+BinnedLRUHandle* BinnedLRUHandleTable::Insert(BinnedLRUHandle* h) {
+ BinnedLRUHandle** ptr = FindPointer(h->key(), h->hash);
+ BinnedLRUHandle* old = *ptr;
+ h->next_hash = (old == nullptr ? nullptr : old->next_hash);
+ *ptr = h;
+ if (old == nullptr) {
+ ++elems_;
+ if (elems_ > length_) {
+ // Since each cache entry is fairly large, we aim for a small
+ // average linked list length (<= 1).
+ Resize();
+ }
+ }
+ return old;
+}
+
+BinnedLRUHandle* BinnedLRUHandleTable::Remove(const rocksdb::Slice& key, uint32_t hash) {
+ BinnedLRUHandle** ptr = FindPointer(key, hash);
+ BinnedLRUHandle* result = *ptr;
+ if (result != nullptr) {
+ *ptr = result->next_hash;
+ --elems_;
+ }
+ return result;
+}
+
+BinnedLRUHandle** BinnedLRUHandleTable::FindPointer(const rocksdb::Slice& key, uint32_t hash) {
+ BinnedLRUHandle** ptr = &list_[hash & (length_ - 1)];
+ while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
+ ptr = &(*ptr)->next_hash;
+ }
+ return ptr;
+}
+
+void BinnedLRUHandleTable::Resize() {
+ uint32_t new_length = 16;
+ while (new_length < elems_ * 1.5) {
+ new_length *= 2;
+ }
+ BinnedLRUHandle** new_list = new BinnedLRUHandle*[new_length];
+ memset(new_list, 0, sizeof(new_list[0]) * new_length);
+ uint32_t count = 0;
+ for (uint32_t i = 0; i < length_; i++) {
+ BinnedLRUHandle* h = list_[i];
+ while (h != nullptr) {
+ BinnedLRUHandle* next = h->next_hash;
+ uint32_t hash = h->hash;
+ BinnedLRUHandle** ptr = &new_list[hash & (new_length - 1)];
+ h->next_hash = *ptr;
+ *ptr = h;
+ h = next;
+ count++;
+ }
+ }
+ ceph_assert(elems_ == count);
+ delete[] list_;
+ list_ = new_list;
+ length_ = new_length;
+}
+
+BinnedLRUCacheShard::BinnedLRUCacheShard(CephContext *c, size_t capacity, bool strict_capacity_limit,
+ double high_pri_pool_ratio)
+ : cct(c),
+ capacity_(0),
+ high_pri_pool_usage_(0),
+ strict_capacity_limit_(strict_capacity_limit),
+ high_pri_pool_ratio_(high_pri_pool_ratio),
+ high_pri_pool_capacity_(0),
+ usage_(0),
+ lru_usage_(0),
+ age_bins(1) {
+ shift_bins();
+ // Make empty circular linked list
+ lru_.next = &lru_;
+ lru_.prev = &lru_;
+ lru_low_pri_ = &lru_;
+ SetCapacity(capacity);
+}
+
+BinnedLRUCacheShard::~BinnedLRUCacheShard() {}
+
+bool BinnedLRUCacheShard::Unref(BinnedLRUHandle* e) {
+ ceph_assert(e->refs > 0);
+ e->refs--;
+ return e->refs == 0;
+}
+
+// Call deleter and free
+
+void BinnedLRUCacheShard::EraseUnRefEntries() {
+ ceph::autovector<BinnedLRUHandle*> last_reference_list;
+ {
+ std::lock_guard<std::mutex> l(mutex_);
+ while (lru_.next != &lru_) {
+ BinnedLRUHandle* old = lru_.next;
+ ceph_assert(old->InCache());
+ ceph_assert(old->refs ==
+ 1); // LRU list contains elements which may be evicted
+ LRU_Remove(old);
+ table_.Remove(old->key(), old->hash);
+ old->SetInCache(false);
+ Unref(old);
+ usage_ -= old->charge;
+ last_reference_list.push_back(old);
+ }
+ }
+
+ for (auto entry : last_reference_list) {
+ entry->Free();
+ }
+}
+
+void BinnedLRUCacheShard::ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe)
+{
+ if (thread_safe) {
+ mutex_.lock();
+ }
+ table_.ApplyToAllCacheEntries(
+ [callback](BinnedLRUHandle* h) {
+ callback(h->key(), h->value, h->charge, h->deleter);
+ });
+ if (thread_safe) {
+ mutex_.unlock();
+ }
+}
+
+void BinnedLRUCacheShard::TEST_GetLRUList(BinnedLRUHandle** lru, BinnedLRUHandle** lru_low_pri) {
+ *lru = &lru_;
+ *lru_low_pri = lru_low_pri_;
+}
+
+size_t BinnedLRUCacheShard::TEST_GetLRUSize() {
+ BinnedLRUHandle* lru_handle = lru_.next;
+ size_t lru_size = 0;
+ while (lru_handle != &lru_) {
+ lru_size++;
+ lru_handle = lru_handle->next;
+ }
+ return lru_size;
+}
+
+double BinnedLRUCacheShard::GetHighPriPoolRatio() const {
+ std::lock_guard<std::mutex> l(mutex_);
+ return high_pri_pool_ratio_;
+}
+
+size_t BinnedLRUCacheShard::GetHighPriPoolUsage() const {
+ std::lock_guard<std::mutex> l(mutex_);
+ return high_pri_pool_usage_;
+}
+
+void BinnedLRUCacheShard::LRU_Remove(BinnedLRUHandle* e) {
+ ceph_assert(e->next != nullptr);
+ ceph_assert(e->prev != nullptr);
+ if (lru_low_pri_ == e) {
+ lru_low_pri_ = e->prev;
+ }
+ e->next->prev = e->prev;
+ e->prev->next = e->next;
+ e->prev = e->next = nullptr;
+ lru_usage_ -= e->charge;
+ if (e->InHighPriPool()) {
+ ceph_assert(high_pri_pool_usage_ >= e->charge);
+ high_pri_pool_usage_ -= e->charge;
+ } else {
+ ceph_assert(*(e->age_bin) >= e->charge);
+ *(e->age_bin) -= e->charge;
+ }
+}
+
+void BinnedLRUCacheShard::LRU_Insert(BinnedLRUHandle* e) {
+ ceph_assert(e->next == nullptr);
+ ceph_assert(e->prev == nullptr);
+ e->age_bin = age_bins.front();
+
+ if (high_pri_pool_ratio_ > 0 && e->IsHighPri()) {
+ // Inset "e" to head of LRU list.
+ e->next = &lru_;
+ e->prev = lru_.prev;
+ e->prev->next = e;
+ e->next->prev = e;
+ e->SetInHighPriPool(true);
+ high_pri_pool_usage_ += e->charge;
+ MaintainPoolSize();
+ } else {
+ // Insert "e" to the head of low-pri pool. Note that when
+ // high_pri_pool_ratio is 0, head of low-pri pool is also head of LRU list.
+ e->next = lru_low_pri_->next;
+ e->prev = lru_low_pri_;
+ e->prev->next = e;
+ e->next->prev = e;
+ e->SetInHighPriPool(false);
+ lru_low_pri_ = e;
+ *(e->age_bin) += e->charge;
+ }
+ lru_usage_ += e->charge;
+}
+
+uint64_t BinnedLRUCacheShard::sum_bins(uint32_t start, uint32_t end) const {
+ std::lock_guard<std::mutex> l(mutex_);
+ auto size = age_bins.size();
+ if (size < start) {
+ return 0;
+ }
+ uint64_t bytes = 0;
+ end = (size < end) ? size : end;
+ for (auto i = start; i < end; i++) {
+ bytes += *(age_bins[i]);
+ }
+ return bytes;
+}
+
+void BinnedLRUCacheShard::MaintainPoolSize() {
+ while (high_pri_pool_usage_ > high_pri_pool_capacity_) {
+ // Overflow last entry in high-pri pool to low-pri pool.
+ lru_low_pri_ = lru_low_pri_->next;
+ ceph_assert(lru_low_pri_ != &lru_);
+ lru_low_pri_->SetInHighPriPool(false);
+ high_pri_pool_usage_ -= lru_low_pri_->charge;
+ *(lru_low_pri_->age_bin) += lru_low_pri_->charge;
+ }
+}
+
+void BinnedLRUCacheShard::EvictFromLRU(size_t charge,
+ ceph::autovector<BinnedLRUHandle*>* deleted) {
+ while (usage_ + charge > capacity_ && lru_.next != &lru_) {
+ BinnedLRUHandle* old = lru_.next;
+ ceph_assert(old->InCache());
+ ceph_assert(old->refs == 1); // LRU list contains elements which may be evicted
+ LRU_Remove(old);
+ table_.Remove(old->key(), old->hash);
+ old->SetInCache(false);
+ Unref(old);
+ usage_ -= old->charge;
+ deleted->push_back(old);
+ }
+}
+
+void BinnedLRUCacheShard::SetCapacity(size_t capacity) {
+ ceph::autovector<BinnedLRUHandle*> last_reference_list;
+ {
+ std::lock_guard<std::mutex> l(mutex_);
+ capacity_ = capacity;
+ high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
+ EvictFromLRU(0, &last_reference_list);
+ }
+ // we free the entries here outside of mutex for
+ // performance reasons
+ for (auto entry : last_reference_list) {
+ entry->Free();
+ }
+}
+
+void BinnedLRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
+ std::lock_guard<std::mutex> l(mutex_);
+ strict_capacity_limit_ = strict_capacity_limit;
+}
+
+rocksdb::Cache::Handle* BinnedLRUCacheShard::Lookup(const rocksdb::Slice& key, uint32_t hash) {
+ std::lock_guard<std::mutex> l(mutex_);
+ BinnedLRUHandle* e = table_.Lookup(key, hash);
+ if (e != nullptr) {
+ ceph_assert(e->InCache());
+ if (e->refs == 1) {
+ LRU_Remove(e);
+ }
+ e->refs++;
+ e->SetHit();
+ }
+ return reinterpret_cast<rocksdb::Cache::Handle*>(e);
+}
+
+bool BinnedLRUCacheShard::Ref(rocksdb::Cache::Handle* h) {
+ BinnedLRUHandle* handle = reinterpret_cast<BinnedLRUHandle*>(h);
+ std::lock_guard<std::mutex> l(mutex_);
+ if (handle->InCache() && handle->refs == 1) {
+ LRU_Remove(handle);
+ }
+ handle->refs++;
+ return true;
+}
+
+void BinnedLRUCacheShard::SetHighPriPoolRatio(double high_pri_pool_ratio) {
+ std::lock_guard<std::mutex> l(mutex_);
+ high_pri_pool_ratio_ = high_pri_pool_ratio;
+ high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
+ MaintainPoolSize();
+}
+
+bool BinnedLRUCacheShard::Release(rocksdb::Cache::Handle* handle, bool force_erase) {
+ if (handle == nullptr) {
+ return false;
+ }
+ BinnedLRUHandle* e = reinterpret_cast<BinnedLRUHandle*>(handle);
+ bool last_reference = false;
+ {
+ std::lock_guard<std::mutex> l(mutex_);
+ last_reference = Unref(e);
+ if (last_reference) {
+ usage_ -= e->charge;
+ }
+ if (e->refs == 1 && e->InCache()) {
+ // The item is still in cache, and nobody else holds a reference to it
+ if (usage_ > capacity_ || force_erase) {
+ // the cache is full
+ // The LRU list must be empty since the cache is full
+ ceph_assert(!(usage_ > capacity_) || lru_.next == &lru_);
+ // take this opportunity and remove the item
+ table_.Remove(e->key(), e->hash);
+ e->SetInCache(false);
+ Unref(e);
+ usage_ -= e->charge;
+ last_reference = true;
+ } else {
+ // put the item on the list to be potentially freed
+ LRU_Insert(e);
+ }
+ }
+ }
+
+ // free outside of mutex
+ if (last_reference) {
+ e->Free();
+ }
+ return last_reference;
+}
+
+rocksdb::Status BinnedLRUCacheShard::Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ DeleterFn deleter,
+ rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) {
+ auto e = new BinnedLRUHandle();
+ rocksdb::Status s;
+ ceph::autovector<BinnedLRUHandle*> last_reference_list;
+
+ e->value = value;
+ e->deleter = deleter;
+ e->charge = charge;
+ e->key_length = key.size();
+ e->key_data = new char[e->key_length];
+ e->flags = 0;
+ e->hash = hash;
+ e->refs = (handle == nullptr
+ ? 1
+ : 2); // One from BinnedLRUCache, one for the returned handle
+ e->next = e->prev = nullptr;
+ e->SetInCache(true);
+ e->SetPriority(priority);
+ std::copy_n(key.data(), e->key_length, e->key_data);
+
+ {
+ std::lock_guard<std::mutex> l(mutex_);
+ // Free the space following strict LRU policy until enough space
+ // is freed or the lru list is empty
+ EvictFromLRU(charge, &last_reference_list);
+
+ if (usage_ - lru_usage_ + charge > capacity_ &&
+ (strict_capacity_limit_ || handle == nullptr)) {
+ if (handle == nullptr) {
+ // Don't insert the entry but still return ok, as if the entry inserted
+ // into cache and get evicted immediately.
+ last_reference_list.push_back(e);
+ } else {
+ delete e;
+ *handle = nullptr;
+ s = rocksdb::Status::Incomplete("Insert failed due to LRU cache being full.");
+ }
+ } else {
+ // insert into the cache
+ // note that the cache might get larger than its capacity if not enough
+ // space was freed
+ BinnedLRUHandle* old = table_.Insert(e);
+ usage_ += e->charge;
+ if (old != nullptr) {
+ old->SetInCache(false);
+ if (Unref(old)) {
+ usage_ -= old->charge;
+ // old is on LRU because it's in cache and its reference count
+ // was just 1 (Unref returned 0)
+ LRU_Remove(old);
+ last_reference_list.push_back(old);
+ }
+ }
+ if (handle == nullptr) {
+ LRU_Insert(e);
+ } else {
+ *handle = reinterpret_cast<rocksdb::Cache::Handle*>(e);
+ }
+ s = rocksdb::Status::OK();
+ }
+ }
+
+ // we free the entries here outside of mutex for
+ // performance reasons
+ for (auto entry : last_reference_list) {
+ entry->Free();
+ }
+
+ return s;
+}
+
+void BinnedLRUCacheShard::Erase(const rocksdb::Slice& key, uint32_t hash) {
+ BinnedLRUHandle* e;
+ bool last_reference = false;
+ {
+ std::lock_guard<std::mutex> l(mutex_);
+ e = table_.Remove(key, hash);
+ if (e != nullptr) {
+ last_reference = Unref(e);
+ if (last_reference) {
+ usage_ -= e->charge;
+ }
+ if (last_reference && e->InCache()) {
+ LRU_Remove(e);
+ }
+ e->SetInCache(false);
+ }
+ }
+
+ // mutex not held here
+ // last_reference will only be true if e != nullptr
+ if (last_reference) {
+ e->Free();
+ }
+}
+
+size_t BinnedLRUCacheShard::GetUsage() const {
+ std::lock_guard<std::mutex> l(mutex_);
+ return usage_;
+}
+
+size_t BinnedLRUCacheShard::GetPinnedUsage() const {
+ std::lock_guard<std::mutex> l(mutex_);
+ ceph_assert(usage_ >= lru_usage_);
+ return usage_ - lru_usage_;
+}
+
+void BinnedLRUCacheShard::shift_bins() {
+ std::lock_guard<std::mutex> l(mutex_);
+ age_bins.push_front(std::make_shared<uint64_t>(0));
+}
+
+uint32_t BinnedLRUCacheShard::get_bin_count() const {
+ std::lock_guard<std::mutex> l(mutex_);
+ return age_bins.capacity();
+}
+
+void BinnedLRUCacheShard::set_bin_count(uint32_t count) {
+ std::lock_guard<std::mutex> l(mutex_);
+ age_bins.set_capacity(count);
+}
+
+std::string BinnedLRUCacheShard::GetPrintableOptions() const {
+ const int kBufferSize = 200;
+ char buffer[kBufferSize];
+ {
+ std::lock_guard<std::mutex> l(mutex_);
+ snprintf(buffer, kBufferSize, " high_pri_pool_ratio: %.3lf\n",
+ high_pri_pool_ratio_);
+ }
+ return std::string(buffer);
+}
+
+DeleterFn BinnedLRUCacheShard::GetDeleter(rocksdb::Cache::Handle* h) const
+{
+ auto* handle = reinterpret_cast<BinnedLRUHandle*>(h);
+ return handle->deleter;
+}
+
+BinnedLRUCache::BinnedLRUCache(CephContext *c,
+ size_t capacity,
+ int num_shard_bits,
+ bool strict_capacity_limit,
+ double high_pri_pool_ratio)
+ : ShardedCache(capacity, num_shard_bits, strict_capacity_limit), cct(c) {
+ num_shards_ = 1 << num_shard_bits;
+ // TODO: Switch over to use mempool
+ int rc = posix_memalign((void**) &shards_,
+ CACHE_LINE_SIZE,
+ sizeof(BinnedLRUCacheShard) * num_shards_);
+ if (rc != 0) {
+ throw std::bad_alloc();
+ }
+ size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
+ for (int i = 0; i < num_shards_; i++) {
+ new (&shards_[i])
+ BinnedLRUCacheShard(c, per_shard, strict_capacity_limit, high_pri_pool_ratio);
+ }
+}
+
+BinnedLRUCache::~BinnedLRUCache() {
+ for (int i = 0; i < num_shards_; i++) {
+ shards_[i].~BinnedLRUCacheShard();
+ }
+ aligned_free(shards_);
+}
+
+CacheShard* BinnedLRUCache::GetShard(int shard) {
+ return reinterpret_cast<CacheShard*>(&shards_[shard]);
+}
+
+const CacheShard* BinnedLRUCache::GetShard(int shard) const {
+ return reinterpret_cast<CacheShard*>(&shards_[shard]);
+}
+
+void* BinnedLRUCache::Value(Handle* handle) {
+ return reinterpret_cast<const BinnedLRUHandle*>(handle)->value;
+}
+
+size_t BinnedLRUCache::GetCharge(Handle* handle) const {
+ return reinterpret_cast<const BinnedLRUHandle*>(handle)->charge;
+}
+
+uint32_t BinnedLRUCache::GetHash(Handle* handle) const {
+ return reinterpret_cast<const BinnedLRUHandle*>(handle)->hash;
+}
+
+void BinnedLRUCache::DisownData() {
+// Do not drop data if compile with ASAN to suppress leak warning.
+#ifndef __SANITIZE_ADDRESS__
+ shards_ = nullptr;
+#endif // !__SANITIZE_ADDRESS__
+}
+
+#if (ROCKSDB_MAJOR >= 7 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 22))
+DeleterFn BinnedLRUCache::GetDeleter(Handle* handle) const
+{
+ return reinterpret_cast<const BinnedLRUHandle*>(handle)->deleter;
+}
+#endif
+
+size_t BinnedLRUCache::TEST_GetLRUSize() {
+ size_t lru_size_of_all_shards = 0;
+ for (int i = 0; i < num_shards_; i++) {
+ lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
+ }
+ return lru_size_of_all_shards;
+}
+
+void BinnedLRUCache::SetHighPriPoolRatio(double high_pri_pool_ratio) {
+ for (int i = 0; i < num_shards_; i++) {
+ shards_[i].SetHighPriPoolRatio(high_pri_pool_ratio);
+ }
+}
+
+double BinnedLRUCache::GetHighPriPoolRatio() const {
+ double result = 0.0;
+ if (num_shards_ > 0) {
+ result = shards_[0].GetHighPriPoolRatio();
+ }
+ return result;
+}
+
+size_t BinnedLRUCache::GetHighPriPoolUsage() const {
+ // We will not lock the cache when getting the usage from shards.
+ size_t usage = 0;
+ for (int s = 0; s < num_shards_; s++) {
+ usage += shards_[s].GetHighPriPoolUsage();
+ }
+ return usage;
+}
+
+// PriCache
+
+int64_t BinnedLRUCache::request_cache_bytes(PriorityCache::Priority pri, uint64_t total_cache) const
+{
+ int64_t assigned = get_cache_bytes(pri);
+ int64_t request = 0;
+
+ switch(pri) {
+ // PRI0 is for rocksdb's high priority items (indexes/filters)
+ case PriorityCache::Priority::PRI0:
+ {
+ // Because we want the high pri cache to grow independently of the low
+ // pri cache, request a chunky allocation independent of the other
+ // priorities.
+ request = PriorityCache::get_chunk(GetHighPriPoolUsage(), total_cache);
+ break;
+ }
+ case PriorityCache::Priority::LAST:
+ {
+ auto max = get_bin_count();
+ request = GetUsage();
+ request -= GetHighPriPoolUsage();
+ request -= sum_bins(0, max);
+ break;
+ }
+ default:
+ {
+ ceph_assert(pri > 0 && pri < PriorityCache::Priority::LAST);
+ auto prev_pri = static_cast<PriorityCache::Priority>(pri - 1);
+ uint64_t start = get_bins(prev_pri);
+ uint64_t end = get_bins(pri);
+ request = sum_bins(start, end);
+ break;
+ }
+ }
+ request = (request > assigned) ? request - assigned : 0;
+ ldout(cct, 10) << __func__ << " Priority: " << static_cast<uint32_t>(pri)
+ << " Request: " << request << dendl;
+ return request;
+}
+
+int64_t BinnedLRUCache::commit_cache_size(uint64_t total_bytes)
+{
+ size_t old_bytes = GetCapacity();
+ int64_t new_bytes = PriorityCache::get_chunk(
+ get_cache_bytes(), total_bytes);
+ ldout(cct, 10) << __func__ << " old: " << old_bytes
+ << " new: " << new_bytes << dendl;
+ SetCapacity((size_t) new_bytes);
+
+ double ratio = 0;
+ if (new_bytes > 0) {
+ int64_t pri0_bytes = get_cache_bytes(PriorityCache::Priority::PRI0);
+ ratio = (double) pri0_bytes / new_bytes;
+ }
+ ldout(cct, 5) << __func__ << " High Pri Pool Ratio set to " << ratio << dendl;
+ SetHighPriPoolRatio(ratio);
+ return new_bytes;
+}
+
+void BinnedLRUCache::shift_bins() {
+ for (int s = 0; s < num_shards_; s++) {
+ shards_[s].shift_bins();
+ }
+}
+
+uint64_t BinnedLRUCache::sum_bins(uint32_t start, uint32_t end) const {
+ uint64_t bytes = 0;
+ for (int s = 0; s < num_shards_; s++) {
+ bytes += shards_[s].sum_bins(start, end);
+ }
+ return bytes;
+}
+
+uint32_t BinnedLRUCache::get_bin_count() const {
+ uint32_t result = 0;
+ if (num_shards_ > 0) {
+ result = shards_[0].get_bin_count();
+ }
+ return result;
+}
+
+void BinnedLRUCache::set_bin_count(uint32_t count) {
+ for (int s = 0; s < num_shards_; s++) {
+ shards_[s].set_bin_count(count);
+ }
+}
+
+std::shared_ptr<rocksdb::Cache> NewBinnedLRUCache(
+ CephContext *c,
+ size_t capacity,
+ int num_shard_bits,
+ bool strict_capacity_limit,
+ double high_pri_pool_ratio) {
+ if (num_shard_bits >= 20) {
+ return nullptr; // the cache cannot be sharded into too many fine pieces
+ }
+ if (high_pri_pool_ratio < 0.0 || high_pri_pool_ratio > 1.0) {
+ // invalid high_pri_pool_ratio
+ return nullptr;
+ }
+ if (num_shard_bits < 0) {
+ num_shard_bits = GetDefaultCacheShardBits(capacity);
+ }
+ return std::make_shared<BinnedLRUCache>(
+ c, capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio);
+}
+
+} // namespace rocksdb_cache
diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.h b/src/kv/rocksdb_cache/BinnedLRUCache.h
new file mode 100644
index 000000000..d6d109d9d
--- /dev/null
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.h
@@ -0,0 +1,366 @@
+// Copyright (c) 2018-Present Red Hat Inc. All rights reserved.
+//
+// Copyright (c) 2011-2018, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 and Apache 2.0 License
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef ROCKSDB_BINNED_LRU_CACHE
+#define ROCKSDB_BINNED_LRU_CACHE
+
+#include <string>
+#include <mutex>
+#include <boost/circular_buffer.hpp>
+
+#include "ShardedCache.h"
+#include "common/autovector.h"
+#include "common/dout.h"
+#include "include/ceph_assert.h"
+#include "common/ceph_context.h"
+
+namespace rocksdb_cache {
+
+// LRU cache implementation
+
+// An entry is a variable length heap-allocated structure.
+// Entries are referenced by cache and/or by any external entity.
+// The cache keeps all its entries in table. Some elements
+// are also stored on LRU list.
+//
+// BinnedLRUHandle can be in these states:
+// 1. Referenced externally AND in hash table.
+// In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true)
+// 2. Not referenced externally and in hash table. In that case the entry is
+// in the LRU and can be freed. (refs == 1 && in_cache == true)
+// 3. Referenced externally and not in hash table. In that case the entry is
+// in not on LRU and not in table. (refs >= 1 && in_cache == false)
+//
+// All newly created BinnedLRUHandles are in state 1. If you call
+// BinnedLRUCacheShard::Release
+// on entry in state 1, it will go into state 2. To move from state 1 to
+// state 3, either call BinnedLRUCacheShard::Erase or BinnedLRUCacheShard::Insert with the
+// same key.
+// To move from state 2 to state 1, use BinnedLRUCacheShard::Lookup.
+// Before destruction, make sure that no handles are in state 1. This means
+// that any successful BinnedLRUCacheShard::Lookup/BinnedLRUCacheShard::Insert have a
+// matching
+// RUCache::Release (to move into state 2) or BinnedLRUCacheShard::Erase (for state 3)
+
+std::shared_ptr<rocksdb::Cache> NewBinnedLRUCache(
+ CephContext *c,
+ size_t capacity,
+ int num_shard_bits = -1,
+ bool strict_capacity_limit = false,
+ double high_pri_pool_ratio = 0.0);
+
+struct BinnedLRUHandle {
+ std::shared_ptr<uint64_t> age_bin;
+ void* value;
+ DeleterFn deleter;
+ BinnedLRUHandle* next_hash;
+ BinnedLRUHandle* next;
+ BinnedLRUHandle* prev;
+ size_t charge; // TODO(opt): Only allow uint32_t?
+ size_t key_length;
+ uint32_t refs; // a number of refs to this entry
+ // cache itself is counted as 1
+
+ // Include the following flags:
+ // in_cache: whether this entry is referenced by the hash table.
+ // is_high_pri: whether this entry is high priority entry.
+ // in_high_pri_pool: whether this entry is in high-pri pool.
+ char flags;
+
+ uint32_t hash; // Hash of key(); used for fast sharding and comparisons
+
+ char* key_data = nullptr; // Beginning of key
+
+ rocksdb::Slice key() const {
+ // For cheaper lookups, we allow a temporary Handle object
+ // to store a pointer to a key in "value".
+ if (next == this) {
+ return *(reinterpret_cast<rocksdb::Slice*>(value));
+ } else {
+ return rocksdb::Slice(key_data, key_length);
+ }
+ }
+
+ bool InCache() { return flags & 1; }
+ bool IsHighPri() { return flags & 2; }
+ bool InHighPriPool() { return flags & 4; }
+ bool HasHit() { return flags & 8; }
+
+ void SetInCache(bool in_cache) {
+ if (in_cache) {
+ flags |= 1;
+ } else {
+ flags &= ~1;
+ }
+ }
+
+ void SetPriority(rocksdb::Cache::Priority priority) {
+ if (priority == rocksdb::Cache::Priority::HIGH) {
+ flags |= 2;
+ } else {
+ flags &= ~2;
+ }
+ }
+
+ void SetInHighPriPool(bool in_high_pri_pool) {
+ if (in_high_pri_pool) {
+ flags |= 4;
+ } else {
+ flags &= ~4;
+ }
+ }
+
+ void SetHit() { flags |= 8; }
+
+ void Free() {
+ ceph_assert((refs == 1 && InCache()) || (refs == 0 && !InCache()));
+ if (deleter) {
+ (*deleter)(key(), value);
+ }
+ delete[] key_data;
+ delete this;
+ }
+};
+
+// We provide our own simple hash table since it removes a whole bunch
+// of porting hacks and is also faster than some of the built-in hash
+// table implementations in some of the compiler/runtime combinations
+// we have tested. E.g., readrandom speeds up by ~5% over the g++
+// 4.4.3's builtin hashtable.
+class BinnedLRUHandleTable {
+ public:
+ BinnedLRUHandleTable();
+ ~BinnedLRUHandleTable();
+
+ BinnedLRUHandle* Lookup(const rocksdb::Slice& key, uint32_t hash);
+ BinnedLRUHandle* Insert(BinnedLRUHandle* h);
+ BinnedLRUHandle* Remove(const rocksdb::Slice& key, uint32_t hash);
+
+ template <typename T>
+ void ApplyToAllCacheEntries(T func) {
+ for (uint32_t i = 0; i < length_; i++) {
+ BinnedLRUHandle* h = list_[i];
+ while (h != nullptr) {
+ auto n = h->next_hash;
+ ceph_assert(h->InCache());
+ func(h);
+ h = n;
+ }
+ }
+ }
+
+ private:
+ // Return a pointer to slot that points to a cache entry that
+ // matches key/hash. If there is no such cache entry, return a
+ // pointer to the trailing slot in the corresponding linked list.
+ BinnedLRUHandle** FindPointer(const rocksdb::Slice& key, uint32_t hash);
+
+ void Resize();
+
+ // The table consists of an array of buckets where each bucket is
+ // a linked list of cache entries that hash into the bucket.
+ BinnedLRUHandle** list_;
+ uint32_t length_;
+ uint32_t elems_;
+};
+
+// A single shard of sharded cache.
+class alignas(CACHE_LINE_SIZE) BinnedLRUCacheShard : public CacheShard {
+ public:
+ BinnedLRUCacheShard(CephContext *c, size_t capacity, bool strict_capacity_limit,
+ double high_pri_pool_ratio);
+ virtual ~BinnedLRUCacheShard();
+
+ // Separate from constructor so caller can easily make an array of BinnedLRUCache
+ // if current usage is more than new capacity, the function will attempt to
+ // free the needed space
+ virtual void SetCapacity(size_t capacity) override;
+
+ // Set the flag to reject insertion if cache if full.
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+
+ // Set percentage of capacity reserved for high-pri cache entries.
+ void SetHighPriPoolRatio(double high_pri_pool_ratio);
+
+ // Like Cache methods, but with an extra "hash" parameter.
+ virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ DeleterFn deleter,
+ rocksdb::Cache::Handle** handle,
+ rocksdb::Cache::Priority priority) override;
+ virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) override;
+ virtual bool Ref(rocksdb::Cache::Handle* handle) override;
+ virtual bool Release(rocksdb::Cache::Handle* handle,
+ bool force_erase = false) override;
+ virtual void Erase(const rocksdb::Slice& key, uint32_t hash) override;
+
+ // Although in some platforms the update of size_t is atomic, to make sure
+ // GetUsage() and GetPinnedUsage() work correctly under any platform, we'll
+ // protect them with mutex_.
+
+ virtual size_t GetUsage() const override;
+ virtual size_t GetPinnedUsage() const override;
+
+ virtual void ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe) override;
+
+ virtual void EraseUnRefEntries() override;
+
+ virtual std::string GetPrintableOptions() const override;
+
+ virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const override;
+
+ void TEST_GetLRUList(BinnedLRUHandle** lru, BinnedLRUHandle** lru_low_pri);
+
+ // Retrieves number of elements in LRU, for unit test purpose only
+ // not threadsafe
+ size_t TEST_GetLRUSize();
+
+ // Retrieves high pri pool ratio
+ double GetHighPriPoolRatio() const;
+
+ // Retrieves high pri pool usage
+ size_t GetHighPriPoolUsage() const;
+
+ // Rotate the bins
+ void shift_bins();
+
+ // Get the bin count
+ uint32_t get_bin_count() const;
+
+ // Set the bin count
+ void set_bin_count(uint32_t count);
+
+ // Get the byte counts for a range of age bins
+ uint64_t sum_bins(uint32_t start, uint32_t end) const;
+
+ private:
+ CephContext *cct;
+ void LRU_Remove(BinnedLRUHandle* e);
+ void LRU_Insert(BinnedLRUHandle* e);
+
+ // Overflow the last entry in high-pri pool to low-pri pool until size of
+ // high-pri pool is no larger than the size specify by high_pri_pool_pct.
+ void MaintainPoolSize();
+
+ // Just reduce the reference count by 1.
+ // Return true if last reference
+ bool Unref(BinnedLRUHandle* e);
+
+ // Free some space following strict LRU policy until enough space
+ // to hold (usage_ + charge) is freed or the lru list is empty
+ // This function is not thread safe - it needs to be executed while
+ // holding the mutex_
+ void EvictFromLRU(size_t charge, ceph::autovector<BinnedLRUHandle*>* deleted);
+
+ // Initialized before use.
+ size_t capacity_;
+
+ // Memory size for entries in high-pri pool.
+ size_t high_pri_pool_usage_;
+
+ // Whether to reject insertion if cache reaches its full capacity.
+ bool strict_capacity_limit_;
+
+ // Ratio of capacity reserved for high priority cache entries.
+ double high_pri_pool_ratio_;
+
+ // High-pri pool size, equals to capacity * high_pri_pool_ratio.
+ // Remember the value to avoid recomputing each time.
+ double high_pri_pool_capacity_;
+
+ // Dummy head of LRU list.
+ // lru.prev is newest entry, lru.next is oldest entry.
+ // LRU contains items which can be evicted, ie reference only by cache
+ BinnedLRUHandle lru_;
+
+ // Pointer to head of low-pri pool in LRU list.
+ BinnedLRUHandle* lru_low_pri_;
+
+ // ------------^^^^^^^^^^^^^-----------
+ // Not frequently modified data members
+ // ------------------------------------
+ //
+ // We separate data members that are updated frequently from the ones that
+ // are not frequently updated so that they don't share the same cache line
+ // which will lead into false cache sharing
+ //
+ // ------------------------------------
+ // Frequently modified data members
+ // ------------vvvvvvvvvvvvv-----------
+ BinnedLRUHandleTable table_;
+
+ // Memory size for entries residing in the cache
+ size_t usage_;
+
+ // Memory size for entries residing only in the LRU list
+ size_t lru_usage_;
+
+ // mutex_ protects the following state.
+ // We don't count mutex_ as the cache's internal state so semantically we
+ // don't mind mutex_ invoking the non-const actions.
+ mutable std::mutex mutex_;
+
+ // Circular buffer of byte counters for age binning
+ boost::circular_buffer<std::shared_ptr<uint64_t>> age_bins;
+};
+
+class BinnedLRUCache : public ShardedCache {
+ public:
+ BinnedLRUCache(CephContext *c, size_t capacity, int num_shard_bits,
+ bool strict_capacity_limit, double high_pri_pool_ratio);
+ virtual ~BinnedLRUCache();
+ virtual const char* Name() const override { return "BinnedLRUCache"; }
+ virtual CacheShard* GetShard(int shard) override;
+ virtual const CacheShard* GetShard(int shard) const override;
+ virtual void* Value(Handle* handle) override;
+ virtual size_t GetCharge(Handle* handle) const override;
+ virtual uint32_t GetHash(Handle* handle) const override;
+ virtual void DisownData() override;
+#if (ROCKSDB_MAJOR >= 7 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 22))
+ virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
+ // Retrieves number of elements in LRU, for unit test purpose only
+ size_t TEST_GetLRUSize();
+ // Sets the high pri pool ratio
+ void SetHighPriPoolRatio(double high_pri_pool_ratio);
+ // Retrieves high pri pool ratio
+ double GetHighPriPoolRatio() const;
+ // Retrieves high pri pool usage
+ size_t GetHighPriPoolUsage() const;
+
+ // PriorityCache
+ virtual int64_t request_cache_bytes(
+ PriorityCache::Priority pri, uint64_t total_cache) const;
+ virtual int64_t commit_cache_size(uint64_t total_cache);
+ virtual int64_t get_committed_size() const {
+ return GetCapacity();
+ }
+ virtual void shift_bins();
+ uint64_t sum_bins(uint32_t start, uint32_t end) const;
+ uint32_t get_bin_count() const;
+ void set_bin_count(uint32_t count);
+
+ virtual std::string get_cache_name() const {
+ return "RocksDB Binned LRU Cache";
+ }
+
+ private:
+ CephContext *cct;
+ BinnedLRUCacheShard* shards_;
+ int num_shards_ = 0;
+};
+
+} // namespace rocksdb_cache
+
+#endif // ROCKSDB_BINNED_LRU_CACHE
diff --git a/src/kv/rocksdb_cache/ShardedCache.cc b/src/kv/rocksdb_cache/ShardedCache.cc
new file mode 100644
index 000000000..7d160f9c7
--- /dev/null
+++ b/src/kv/rocksdb_cache/ShardedCache.cc
@@ -0,0 +1,182 @@
+// Copyright (c) 2018-Present Red Hat Inc. All rights reserved.
+//
+// Copyright (c) 2011-2018, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 and Apache 2.0 License
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "ShardedCache.h"
+
+#include <string>
+
+namespace rocksdb_cache {
+
+ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
+ bool strict_capacity_limit)
+ : num_shard_bits_(num_shard_bits),
+ capacity_(capacity),
+ strict_capacity_limit_(strict_capacity_limit),
+ last_id_(1) {}
+
+void ShardedCache::SetCapacity(size_t capacity) {
+ int num_shards = 1 << num_shard_bits_;
+ const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
+ std::lock_guard<std::mutex> l(capacity_mutex_);
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->SetCapacity(per_shard);
+ }
+ capacity_ = capacity;
+}
+
+void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
+ int num_shards = 1 << num_shard_bits_;
+ std::lock_guard<std::mutex> l(capacity_mutex_);
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit);
+ }
+ strict_capacity_limit_ = strict_capacity_limit;
+}
+
+rocksdb::Status ShardedCache::Insert(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn deleter,
+ rocksdb::Cache::Handle** handle, Priority priority) {
+ uint32_t hash = HashSlice(key);
+ return GetShard(Shard(hash))
+ ->Insert(key, hash, value, charge, deleter, handle, priority);
+}
+
+rocksdb::Cache::Handle* ShardedCache::Lookup(const rocksdb::Slice& key, rocksdb::Statistics* /*stats*/) {
+ uint32_t hash = HashSlice(key);
+ return GetShard(Shard(hash))->Lookup(key, hash);
+}
+
+bool ShardedCache::Ref(rocksdb::Cache::Handle* handle) {
+ uint32_t hash = GetHash(handle);
+ return GetShard(Shard(hash))->Ref(handle);
+}
+
+bool ShardedCache::Release(rocksdb::Cache::Handle* handle, bool force_erase) {
+ uint32_t hash = GetHash(handle);
+ return GetShard(Shard(hash))->Release(handle, force_erase);
+}
+
+void ShardedCache::Erase(const rocksdb::Slice& key) {
+ uint32_t hash = HashSlice(key);
+ GetShard(Shard(hash))->Erase(key, hash);
+}
+
+uint64_t ShardedCache::NewId() {
+ return last_id_.fetch_add(1, std::memory_order_relaxed);
+}
+
+size_t ShardedCache::GetCapacity() const {
+ std::lock_guard<std::mutex> l(capacity_mutex_);
+ return capacity_;
+}
+
+bool ShardedCache::HasStrictCapacityLimit() const {
+ std::lock_guard<std::mutex> l(capacity_mutex_);
+ return strict_capacity_limit_;
+}
+
+size_t ShardedCache::GetUsage() const {
+ // We will not lock the cache when getting the usage from shards.
+ int num_shards = 1 << num_shard_bits_;
+ size_t usage = 0;
+ for (int s = 0; s < num_shards; s++) {
+ usage += GetShard(s)->GetUsage();
+ }
+ return usage;
+}
+
+size_t ShardedCache::GetUsage(rocksdb::Cache::Handle* handle) const {
+ return GetCharge(handle);
+}
+
+size_t ShardedCache::GetPinnedUsage() const {
+ // We will not lock the cache when getting the usage from shards.
+ int num_shards = 1 << num_shard_bits_;
+ size_t usage = 0;
+ for (int s = 0; s < num_shards; s++) {
+ usage += GetShard(s)->GetPinnedUsage();
+ }
+ return usage;
+}
+
+#if (ROCKSDB_MAJOR >= 7 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 22))
+DeleterFn ShardedCache::GetDeleter(Handle* handle) const
+{
+ uint32_t hash = GetHash(handle);
+ return GetShard(Shard(hash))->GetDeleter(handle);
+}
+
+void ShardedCache::ApplyToAllEntries(
+ const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn deleter)>& callback,
+ const ApplyToAllEntriesOptions& opts)
+{
+ int num_shards = 1 << num_shard_bits_;
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->ApplyToAllCacheEntries(callback, true /* thread_safe */);
+ }
+}
+#else
+void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) {
+ int num_shards = 1 << num_shard_bits_;
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->ApplyToAllCacheEntries(
+ [callback](const rocksdb::Slice&, void* value, size_t charge, DeleterFn) {
+ callback(value, charge);
+ },
+ thread_safe);
+ }
+}
+#endif
+
+void ShardedCache::EraseUnRefEntries() {
+ int num_shards = 1 << num_shard_bits_;
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->EraseUnRefEntries();
+ }
+}
+
+std::string ShardedCache::GetPrintableOptions() const {
+ std::string ret;
+ ret.reserve(20000);
+ const int kBufferSize = 200;
+ char buffer[kBufferSize];
+ {
+ std::lock_guard<std::mutex> l(capacity_mutex_);
+ snprintf(buffer, kBufferSize, " capacity : %zu\n",
+ capacity_);
+ ret.append(buffer);
+ snprintf(buffer, kBufferSize, " num_shard_bits : %d\n", num_shard_bits_);
+ ret.append(buffer);
+ snprintf(buffer, kBufferSize, " strict_capacity_limit : %d\n",
+ strict_capacity_limit_);
+ ret.append(buffer);
+ }
+ ret.append(GetShard(0)->GetPrintableOptions());
+ return ret;
+}
+int GetDefaultCacheShardBits(size_t capacity) {
+ int num_shard_bits = 0;
+ size_t min_shard_size = 512L * 1024L; // Every shard is at least 512KB.
+ size_t num_shards = capacity / min_shard_size;
+ while (num_shards >>= 1) {
+ if (++num_shard_bits >= 6) {
+ // No more than 6.
+ return num_shard_bits;
+ }
+ }
+ return num_shard_bits;
+}
+
+} // namespace rocksdb_cache
diff --git a/src/kv/rocksdb_cache/ShardedCache.h b/src/kv/rocksdb_cache/ShardedCache.h
new file mode 100644
index 000000000..ef4b10d8f
--- /dev/null
+++ b/src/kv/rocksdb_cache/ShardedCache.h
@@ -0,0 +1,197 @@
+// Copyright (c) 2018-Present Red Hat Inc. All rights reserved.
+//
+// Copyright (c) 2011-2018, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 and Apache 2.0 License
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef ROCKSDB_SHARDED_CACHE
+#define ROCKSDB_SHARDED_CACHE
+
+#include <atomic>
+#include <string>
+#include <mutex>
+
+#include "rocksdb/version.h"
+#include "rocksdb/cache.h"
+#include "include/ceph_hash.h"
+#include "common/PriorityCache.h"
+//#include "hash.h"
+
+#ifndef CACHE_LINE_SIZE
+#define CACHE_LINE_SIZE 64 // XXX arch-specific define
+#endif
+
+namespace rocksdb_cache {
+
+using DeleterFn = void (*)(const rocksdb::Slice& key, void* value);
+
+// Single cache shard interface.
+class CacheShard {
+ public:
+ CacheShard() = default;
+ virtual ~CacheShard() = default;
+
+ virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ DeleterFn deleter,
+ rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) = 0;
+ virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) = 0;
+ virtual bool Ref(rocksdb::Cache::Handle* handle) = 0;
+ virtual bool Release(rocksdb::Cache::Handle* handle, bool force_erase = false) = 0;
+ virtual void Erase(const rocksdb::Slice& key, uint32_t hash) = 0;
+ virtual void SetCapacity(size_t capacity) = 0;
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
+ virtual size_t GetUsage() const = 0;
+ virtual size_t GetPinnedUsage() const = 0;
+ virtual void ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe) = 0;
+ virtual void EraseUnRefEntries() = 0;
+ virtual std::string GetPrintableOptions() const { return ""; }
+ virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const = 0;
+};
+
+// Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
+// shards will be created, with capacity split evenly to each of the shards.
+// Keys are sharded by the highest num_shard_bits bits of hash value.
+class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {
+ public:
+ ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
+ virtual ~ShardedCache() = default;
+ // rocksdb::Cache
+ virtual const char* Name() const override = 0;
+ virtual rocksdb::Status Insert(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn,
+ rocksdb::Cache::Handle** handle, Priority priority) override;
+ virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, rocksdb::Statistics* stats) override;
+ virtual bool Ref(rocksdb::Cache::Handle* handle) override;
+ virtual bool Release(rocksdb::Cache::Handle* handle, bool force_erase = false) override;
+ virtual void* Value(Handle* handle) override = 0;
+ virtual void Erase(const rocksdb::Slice& key) override;
+ virtual uint64_t NewId() override;
+ virtual void SetCapacity(size_t capacity) override;
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+ virtual bool HasStrictCapacityLimit() const override;
+ virtual size_t GetCapacity() const override;
+ virtual size_t GetUsage() const override;
+ virtual size_t GetUsage(rocksdb::Cache::Handle* handle) const override;
+ virtual size_t GetPinnedUsage() const override;
+ virtual size_t GetCharge(Handle* handle) const = 0;
+#if (ROCKSDB_MAJOR >= 7 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 22))
+ virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
+ virtual void DisownData() override = 0;
+#if (ROCKSDB_MAJOR >= 7 || (ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR >= 22))
+ virtual void ApplyToAllEntries(
+ const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn deleter)>& callback,
+ const ApplyToAllEntriesOptions& opts) override;
+#else
+ virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) override;
+#endif
+ virtual void EraseUnRefEntries() override;
+ virtual std::string GetPrintableOptions() const override;
+ virtual CacheShard* GetShard(int shard) = 0;
+ virtual const CacheShard* GetShard(int shard) const = 0;
+ virtual uint32_t GetHash(Handle* handle) const = 0;
+
+ int GetNumShardBits() const { return num_shard_bits_; }
+
+ virtual uint32_t get_bin_count() const = 0;
+ virtual void set_bin_count(uint32_t count) = 0;
+
+ // PriCache
+ virtual int64_t get_cache_bytes(PriorityCache::Priority pri) const {
+ return cache_bytes[pri];
+ }
+ virtual int64_t get_cache_bytes() const {
+ int64_t total = 0;
+ for (int i = 0; i < PriorityCache::Priority::LAST + 1; i++) {
+ PriorityCache::Priority pri = static_cast<PriorityCache::Priority>(i);
+ total += get_cache_bytes(pri);
+ }
+ return total;
+ }
+ virtual void set_cache_bytes(PriorityCache::Priority pri, int64_t bytes) {
+ cache_bytes[pri] = bytes;
+ }
+ virtual void add_cache_bytes(PriorityCache::Priority pri, int64_t bytes) {
+ cache_bytes[pri] += bytes;
+ }
+ virtual double get_cache_ratio() const {
+ return cache_ratio;
+ }
+ virtual void set_cache_ratio(double ratio) {
+ cache_ratio = ratio;
+ }
+ virtual uint64_t get_bins(PriorityCache::Priority pri) const {
+ if (pri > PriorityCache::Priority::PRI0 &&
+ pri < PriorityCache::Priority::LAST) {
+ return bins[pri];
+ }
+ return 0;
+ }
+ virtual void set_bins(PriorityCache::Priority pri, uint64_t end_bin) {
+ if (pri <= PriorityCache::Priority::PRI0 ||
+ pri >= PriorityCache::Priority::LAST) {
+ return;
+ }
+ bins[pri] = end_bin;
+ uint64_t max = 0;
+ for (int pri = 1; pri < PriorityCache::Priority::LAST; pri++) {
+ if (bins[pri] > max) {
+ max = bins[pri];
+ }
+ }
+ set_bin_count(max);
+ }
+ virtual void import_bins(const std::vector<uint64_t> &bins_v) {
+ uint64_t max = 0;
+ for (int pri = 1; pri < PriorityCache::Priority::LAST; pri++) {
+ unsigned i = (unsigned) pri - 1;
+ if (i < bins_v.size()) {
+ bins[pri] = bins_v[i];
+ if (bins[pri] > max) {
+ max = bins[pri];
+ }
+ } else {
+ bins[pri] = 0;
+ }
+ }
+ set_bin_count(max);
+ }
+ virtual std::string get_cache_name() const = 0;
+
+ private:
+ static inline uint32_t HashSlice(const rocksdb::Slice& s) {
+ return ceph_str_hash(CEPH_STR_HASH_RJENKINS, s.data(), s.size());
+// return Hash(s.data(), s.size(), 0);
+ }
+
+ uint32_t Shard(uint32_t hash) const {
+ // Note, hash >> 32 yields hash in gcc, not the zero we expect!
+ return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
+ }
+
+ uint64_t bins[PriorityCache::Priority::LAST+1] = {0};
+ int64_t cache_bytes[PriorityCache::Priority::LAST+1] = {0};
+ double cache_ratio = 0;
+
+ int num_shard_bits_;
+ mutable std::mutex capacity_mutex_;
+ size_t capacity_;
+ bool strict_capacity_limit_;
+ std::atomic<uint64_t> last_id_;
+};
+
+extern int GetDefaultCacheShardBits(size_t capacity);
+
+} // namespace rocksdb_cache
+#endif // ROCKSDB_SHARDED_CACHE