summaryrefslogtreecommitdiffstats
path: root/src/rocksdb/options
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:45:59 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:45:59 +0000
commit19fcec84d8d7d21e796c7624e521b60d28ee21ed (patch)
tree42d26aa27d1e3f7c0b8bd3fd14e7d7082f5008dc /src/rocksdb/options
parentInitial commit. (diff)
downloadceph-upstream/16.2.11+ds.tar.xz
ceph-upstream/16.2.11+ds.zip
Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--src/rocksdb/options/cf_options.cc231
-rw-r--r--src/rocksdb/options/cf_options.h269
-rw-r--r--src/rocksdb/options/db_options.cc333
-rw-r--r--src/rocksdb/options/db_options.h118
-rw-r--r--src/rocksdb/options/options.cc623
-rw-r--r--src/rocksdb/options/options_helper.cc2124
-rw-r--r--src/rocksdb/options/options_helper.h233
-rw-r--r--src/rocksdb/options/options_parser.cc839
-rw-r--r--src/rocksdb/options/options_parser.h147
-rw-r--r--src/rocksdb/options/options_sanity_check.cc38
-rw-r--r--src/rocksdb/options/options_sanity_check.h50
-rw-r--r--src/rocksdb/options/options_settable_test.cc492
-rw-r--r--src/rocksdb/options/options_test.cc2004
13 files changed, 7501 insertions, 0 deletions
diff --git a/src/rocksdb/options/cf_options.cc b/src/rocksdb/options/cf_options.cc
new file mode 100644
index 000000000..bc9e301e6
--- /dev/null
+++ b/src/rocksdb/options/cf_options.cc
@@ -0,0 +1,231 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "options/cf_options.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <limits>
+#include <string>
+#include "options/db_options.h"
+#include "port/port.h"
+#include "rocksdb/concurrent_task_limiter.h"
+#include "rocksdb/env.h"
+#include "rocksdb/file_system.h"
+#include "rocksdb/options.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+ImmutableCFOptions::ImmutableCFOptions(const Options& options)
+ : ImmutableCFOptions(ImmutableDBOptions(options), options) {}
+
+ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options,
+ const ColumnFamilyOptions& cf_options)
+ : compaction_style(cf_options.compaction_style),
+ compaction_pri(cf_options.compaction_pri),
+ user_comparator(cf_options.comparator),
+ internal_comparator(InternalKeyComparator(cf_options.comparator)),
+ merge_operator(cf_options.merge_operator.get()),
+ compaction_filter(cf_options.compaction_filter),
+ compaction_filter_factory(cf_options.compaction_filter_factory.get()),
+ min_write_buffer_number_to_merge(
+ cf_options.min_write_buffer_number_to_merge),
+ max_write_buffer_number_to_maintain(
+ cf_options.max_write_buffer_number_to_maintain),
+ max_write_buffer_size_to_maintain(
+ cf_options.max_write_buffer_size_to_maintain),
+ inplace_update_support(cf_options.inplace_update_support),
+ inplace_callback(cf_options.inplace_callback),
+ info_log(db_options.info_log.get()),
+ statistics(db_options.statistics.get()),
+ rate_limiter(db_options.rate_limiter.get()),
+ info_log_level(db_options.info_log_level),
+ env(db_options.env),
+ fs(db_options.fs.get()),
+ allow_mmap_reads(db_options.allow_mmap_reads),
+ allow_mmap_writes(db_options.allow_mmap_writes),
+ db_paths(db_options.db_paths),
+ memtable_factory(cf_options.memtable_factory.get()),
+ table_factory(cf_options.table_factory.get()),
+ table_properties_collector_factories(
+ cf_options.table_properties_collector_factories),
+ advise_random_on_open(db_options.advise_random_on_open),
+ bloom_locality(cf_options.bloom_locality),
+ purge_redundant_kvs_while_flush(
+ cf_options.purge_redundant_kvs_while_flush),
+ use_fsync(db_options.use_fsync),
+ compression_per_level(cf_options.compression_per_level),
+ bottommost_compression(cf_options.bottommost_compression),
+ bottommost_compression_opts(cf_options.bottommost_compression_opts),
+ compression_opts(cf_options.compression_opts),
+ level_compaction_dynamic_level_bytes(
+ cf_options.level_compaction_dynamic_level_bytes),
+ access_hint_on_compaction_start(
+ db_options.access_hint_on_compaction_start),
+ new_table_reader_for_compaction_inputs(
+ db_options.new_table_reader_for_compaction_inputs),
+ num_levels(cf_options.num_levels),
+ optimize_filters_for_hits(cf_options.optimize_filters_for_hits),
+ force_consistency_checks(cf_options.force_consistency_checks),
+ allow_ingest_behind(db_options.allow_ingest_behind),
+ preserve_deletes(db_options.preserve_deletes),
+ listeners(db_options.listeners),
+ row_cache(db_options.row_cache),
+ max_subcompactions(db_options.max_subcompactions),
+ memtable_insert_with_hint_prefix_extractor(
+ cf_options.memtable_insert_with_hint_prefix_extractor.get()),
+ cf_paths(cf_options.cf_paths),
+ compaction_thread_limiter(cf_options.compaction_thread_limiter),
+ sst_file_checksum_func(db_options.sst_file_checksum_func.get()) {}
+
+// Multiple two operands. If they overflow, return op1.
+uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
+ if (op1 == 0 || op2 <= 0) {
+ return 0;
+ }
+ if (port::kMaxUint64 / op1 < op2) {
+ return op1;
+ }
+ return static_cast<uint64_t>(op1 * op2);
+}
+
+// when level_compaction_dynamic_level_bytes is true and leveled compaction
+// is used, the base level is not always L1, so precomupted max_file_size can
+// no longer be used. Recompute file_size_for_level from base level.
+uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
+ int level, CompactionStyle compaction_style, int base_level,
+ bool level_compaction_dynamic_level_bytes) {
+ if (!level_compaction_dynamic_level_bytes || level < base_level ||
+ compaction_style != kCompactionStyleLevel) {
+ assert(level >= 0);
+ assert(level < (int)cf_options.max_file_size.size());
+ return cf_options.max_file_size[level];
+ } else {
+ assert(level >= 0 && base_level >= 0);
+ assert(level - base_level < (int)cf_options.max_file_size.size());
+ return cf_options.max_file_size[level - base_level];
+ }
+}
+
+void MutableCFOptions::RefreshDerivedOptions(int num_levels,
+ CompactionStyle compaction_style) {
+ max_file_size.resize(num_levels);
+ for (int i = 0; i < num_levels; ++i) {
+ if (i == 0 && compaction_style == kCompactionStyleUniversal) {
+ max_file_size[i] = ULLONG_MAX;
+ } else if (i > 1) {
+ max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1],
+ target_file_size_multiplier);
+ } else {
+ max_file_size[i] = target_file_size_base;
+ }
+ }
+}
+
+void MutableCFOptions::Dump(Logger* log) const {
+ // Memtable related options
+ ROCKS_LOG_INFO(log,
+ " write_buffer_size: %" ROCKSDB_PRIszt,
+ write_buffer_size);
+ ROCKS_LOG_INFO(log, " max_write_buffer_number: %d",
+ max_write_buffer_number);
+ ROCKS_LOG_INFO(log,
+ " arena_block_size: %" ROCKSDB_PRIszt,
+ arena_block_size);
+ ROCKS_LOG_INFO(log, " memtable_prefix_bloom_ratio: %f",
+ memtable_prefix_bloom_size_ratio);
+ ROCKS_LOG_INFO(log, " memtable_whole_key_filtering: %d",
+ memtable_whole_key_filtering);
+ ROCKS_LOG_INFO(log,
+ " memtable_huge_page_size: %" ROCKSDB_PRIszt,
+ memtable_huge_page_size);
+ ROCKS_LOG_INFO(log,
+ " max_successive_merges: %" ROCKSDB_PRIszt,
+ max_successive_merges);
+ ROCKS_LOG_INFO(log,
+ " inplace_update_num_locks: %" ROCKSDB_PRIszt,
+ inplace_update_num_locks);
+ ROCKS_LOG_INFO(
+ log, " prefix_extractor: %s",
+ prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
+ ROCKS_LOG_INFO(log, " disable_auto_compactions: %d",
+ disable_auto_compactions);
+ ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
+ soft_pending_compaction_bytes_limit);
+ ROCKS_LOG_INFO(log, " hard_pending_compaction_bytes_limit: %" PRIu64,
+ hard_pending_compaction_bytes_limit);
+ ROCKS_LOG_INFO(log, " level0_file_num_compaction_trigger: %d",
+ level0_file_num_compaction_trigger);
+ ROCKS_LOG_INFO(log, " level0_slowdown_writes_trigger: %d",
+ level0_slowdown_writes_trigger);
+ ROCKS_LOG_INFO(log, " level0_stop_writes_trigger: %d",
+ level0_stop_writes_trigger);
+ ROCKS_LOG_INFO(log, " max_compaction_bytes: %" PRIu64,
+ max_compaction_bytes);
+ ROCKS_LOG_INFO(log, " target_file_size_base: %" PRIu64,
+ target_file_size_base);
+ ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d",
+ target_file_size_multiplier);
+ ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64,
+ max_bytes_for_level_base);
+ ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f",
+ max_bytes_for_level_multiplier);
+ ROCKS_LOG_INFO(log, " ttl: %" PRIu64,
+ ttl);
+ ROCKS_LOG_INFO(log, " periodic_compaction_seconds: %" PRIu64,
+ periodic_compaction_seconds);
+ std::string result;
+ char buf[10];
+ for (const auto m : max_bytes_for_level_multiplier_additional) {
+ snprintf(buf, sizeof(buf), "%d, ", m);
+ result += buf;
+ }
+ if (result.size() >= 2) {
+ result.resize(result.size() - 2);
+ } else {
+ result = "";
+ }
+
+ ROCKS_LOG_INFO(log, "max_bytes_for_level_multiplier_additional: %s",
+ result.c_str());
+ ROCKS_LOG_INFO(log, " max_sequential_skip_in_iterations: %" PRIu64,
+ max_sequential_skip_in_iterations);
+ ROCKS_LOG_INFO(log, " paranoid_file_checks: %d",
+ paranoid_file_checks);
+ ROCKS_LOG_INFO(log, " report_bg_io_stats: %d",
+ report_bg_io_stats);
+ ROCKS_LOG_INFO(log, " compression: %d",
+ static_cast<int>(compression));
+
+ // Universal Compaction Options
+ ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d",
+ compaction_options_universal.size_ratio);
+ ROCKS_LOG_INFO(log, "compaction_options_universal.min_merge_width : %d",
+ compaction_options_universal.min_merge_width);
+ ROCKS_LOG_INFO(log, "compaction_options_universal.max_merge_width : %d",
+ compaction_options_universal.max_merge_width);
+ ROCKS_LOG_INFO(
+ log, "compaction_options_universal.max_size_amplification_percent : %d",
+ compaction_options_universal.max_size_amplification_percent);
+ ROCKS_LOG_INFO(log,
+ "compaction_options_universal.compression_size_percent : %d",
+ compaction_options_universal.compression_size_percent);
+ ROCKS_LOG_INFO(log, "compaction_options_universal.stop_style : %d",
+ compaction_options_universal.stop_style);
+ ROCKS_LOG_INFO(
+ log, "compaction_options_universal.allow_trivial_move : %d",
+ static_cast<int>(compaction_options_universal.allow_trivial_move));
+
+ // FIFO Compaction Options
+ ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64,
+ compaction_options_fifo.max_table_files_size);
+ ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d",
+ compaction_options_fifo.allow_compaction);
+}
+
+MutableCFOptions::MutableCFOptions(const Options& options)
+ : MutableCFOptions(ColumnFamilyOptions(options)) {}
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/cf_options.h b/src/rocksdb/options/cf_options.h
new file mode 100644
index 000000000..b53ed840f
--- /dev/null
+++ b/src/rocksdb/options/cf_options.h
@@ -0,0 +1,269 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "db/dbformat.h"
+#include "options/db_options.h"
+#include "rocksdb/options.h"
+#include "util/compression.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// ImmutableCFOptions is a data struct used by RocksDB internal. It contains a
+// subset of Options that should not be changed during the entire lifetime
+// of DB. Raw pointers defined in this struct do not have ownership to the data
+// they point to. Options contains std::shared_ptr to these data.
+struct ImmutableCFOptions {
+ explicit ImmutableCFOptions(const Options& options);
+
+ ImmutableCFOptions(const ImmutableDBOptions& db_options,
+ const ColumnFamilyOptions& cf_options);
+
+ CompactionStyle compaction_style;
+
+ CompactionPri compaction_pri;
+
+ const Comparator* user_comparator;
+ InternalKeyComparator internal_comparator;
+
+ MergeOperator* merge_operator;
+
+ const CompactionFilter* compaction_filter;
+
+ CompactionFilterFactory* compaction_filter_factory;
+
+ int min_write_buffer_number_to_merge;
+
+ int max_write_buffer_number_to_maintain;
+
+ int64_t max_write_buffer_size_to_maintain;
+
+ bool inplace_update_support;
+
+ UpdateStatus (*inplace_callback)(char* existing_value,
+ uint32_t* existing_value_size,
+ Slice delta_value,
+ std::string* merged_value);
+
+ Logger* info_log;
+
+ Statistics* statistics;
+
+ RateLimiter* rate_limiter;
+
+ InfoLogLevel info_log_level;
+
+ Env* env;
+
+ FileSystem* fs;
+
+ // Allow the OS to mmap file for reading sst tables. Default: false
+ bool allow_mmap_reads;
+
+ // Allow the OS to mmap file for writing. Default: false
+ bool allow_mmap_writes;
+
+ std::vector<DbPath> db_paths;
+
+ MemTableRepFactory* memtable_factory;
+
+ TableFactory* table_factory;
+
+ Options::TablePropertiesCollectorFactories
+ table_properties_collector_factories;
+
+ bool advise_random_on_open;
+
+ // This options is required by PlainTableReader. May need to move it
+ // to PlainTableOptions just like bloom_bits_per_key
+ uint32_t bloom_locality;
+
+ bool purge_redundant_kvs_while_flush;
+
+ bool use_fsync;
+
+ std::vector<CompressionType> compression_per_level;
+
+ CompressionType bottommost_compression;
+
+ CompressionOptions bottommost_compression_opts;
+
+ CompressionOptions compression_opts;
+
+ bool level_compaction_dynamic_level_bytes;
+
+ Options::AccessHint access_hint_on_compaction_start;
+
+ bool new_table_reader_for_compaction_inputs;
+
+ int num_levels;
+
+ bool optimize_filters_for_hits;
+
+ bool force_consistency_checks;
+
+ bool allow_ingest_behind;
+
+ bool preserve_deletes;
+
+ // A vector of EventListeners which callback functions will be called
+ // when specific RocksDB event happens.
+ std::vector<std::shared_ptr<EventListener>> listeners;
+
+ std::shared_ptr<Cache> row_cache;
+
+ uint32_t max_subcompactions;
+
+ const SliceTransform* memtable_insert_with_hint_prefix_extractor;
+
+ std::vector<DbPath> cf_paths;
+
+ std::shared_ptr<ConcurrentTaskLimiter> compaction_thread_limiter;
+
+ FileChecksumFunc* sst_file_checksum_func;
+};
+
+struct MutableCFOptions {
+ explicit MutableCFOptions(const ColumnFamilyOptions& options)
+ : write_buffer_size(options.write_buffer_size),
+ max_write_buffer_number(options.max_write_buffer_number),
+ arena_block_size(options.arena_block_size),
+ memtable_prefix_bloom_size_ratio(
+ options.memtable_prefix_bloom_size_ratio),
+ memtable_whole_key_filtering(options.memtable_whole_key_filtering),
+ memtable_huge_page_size(options.memtable_huge_page_size),
+ max_successive_merges(options.max_successive_merges),
+ inplace_update_num_locks(options.inplace_update_num_locks),
+ prefix_extractor(options.prefix_extractor),
+ disable_auto_compactions(options.disable_auto_compactions),
+ soft_pending_compaction_bytes_limit(
+ options.soft_pending_compaction_bytes_limit),
+ hard_pending_compaction_bytes_limit(
+ options.hard_pending_compaction_bytes_limit),
+ level0_file_num_compaction_trigger(
+ options.level0_file_num_compaction_trigger),
+ level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
+ level0_stop_writes_trigger(options.level0_stop_writes_trigger),
+ max_compaction_bytes(options.max_compaction_bytes),
+ target_file_size_base(options.target_file_size_base),
+ target_file_size_multiplier(options.target_file_size_multiplier),
+ max_bytes_for_level_base(options.max_bytes_for_level_base),
+ max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
+ ttl(options.ttl),
+ periodic_compaction_seconds(options.periodic_compaction_seconds),
+ max_bytes_for_level_multiplier_additional(
+ options.max_bytes_for_level_multiplier_additional),
+ compaction_options_fifo(options.compaction_options_fifo),
+ compaction_options_universal(options.compaction_options_universal),
+ max_sequential_skip_in_iterations(
+ options.max_sequential_skip_in_iterations),
+ paranoid_file_checks(options.paranoid_file_checks),
+ report_bg_io_stats(options.report_bg_io_stats),
+ compression(options.compression),
+ sample_for_compression(options.sample_for_compression) {
+ RefreshDerivedOptions(options.num_levels, options.compaction_style);
+ }
+
+ MutableCFOptions()
+ : write_buffer_size(0),
+ max_write_buffer_number(0),
+ arena_block_size(0),
+ memtable_prefix_bloom_size_ratio(0),
+ memtable_whole_key_filtering(false),
+ memtable_huge_page_size(0),
+ max_successive_merges(0),
+ inplace_update_num_locks(0),
+ prefix_extractor(nullptr),
+ disable_auto_compactions(false),
+ soft_pending_compaction_bytes_limit(0),
+ hard_pending_compaction_bytes_limit(0),
+ level0_file_num_compaction_trigger(0),
+ level0_slowdown_writes_trigger(0),
+ level0_stop_writes_trigger(0),
+ max_compaction_bytes(0),
+ target_file_size_base(0),
+ target_file_size_multiplier(0),
+ max_bytes_for_level_base(0),
+ max_bytes_for_level_multiplier(0),
+ ttl(0),
+ periodic_compaction_seconds(0),
+ compaction_options_fifo(),
+ max_sequential_skip_in_iterations(0),
+ paranoid_file_checks(false),
+ report_bg_io_stats(false),
+ compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
+ sample_for_compression(0) {}
+
+ explicit MutableCFOptions(const Options& options);
+
+ // Must be called after any change to MutableCFOptions
+ void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style);
+
+ void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) {
+ RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style);
+ }
+
+ int MaxBytesMultiplerAdditional(int level) const {
+ if (level >=
+ static_cast<int>(max_bytes_for_level_multiplier_additional.size())) {
+ return 1;
+ }
+ return max_bytes_for_level_multiplier_additional[level];
+ }
+
+ void Dump(Logger* log) const;
+
+ // Memtable related options
+ size_t write_buffer_size;
+ int max_write_buffer_number;
+ size_t arena_block_size;
+ double memtable_prefix_bloom_size_ratio;
+ bool memtable_whole_key_filtering;
+ size_t memtable_huge_page_size;
+ size_t max_successive_merges;
+ size_t inplace_update_num_locks;
+ std::shared_ptr<const SliceTransform> prefix_extractor;
+
+ // Compaction related options
+ bool disable_auto_compactions;
+ uint64_t soft_pending_compaction_bytes_limit;
+ uint64_t hard_pending_compaction_bytes_limit;
+ int level0_file_num_compaction_trigger;
+ int level0_slowdown_writes_trigger;
+ int level0_stop_writes_trigger;
+ uint64_t max_compaction_bytes;
+ uint64_t target_file_size_base;
+ int target_file_size_multiplier;
+ uint64_t max_bytes_for_level_base;
+ double max_bytes_for_level_multiplier;
+ uint64_t ttl;
+ uint64_t periodic_compaction_seconds;
+ std::vector<int> max_bytes_for_level_multiplier_additional;
+ CompactionOptionsFIFO compaction_options_fifo;
+ CompactionOptionsUniversal compaction_options_universal;
+
+ // Misc options
+ uint64_t max_sequential_skip_in_iterations;
+ bool paranoid_file_checks;
+ bool report_bg_io_stats;
+ CompressionType compression;
+ uint64_t sample_for_compression;
+
+ // Derived options
+ // Per-level target file size.
+ std::vector<uint64_t> max_file_size;
+};
+
+uint64_t MultiplyCheckOverflow(uint64_t op1, double op2);
+
+// Get the max file size in a given level.
+uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
+ int level, CompactionStyle compaction_style, int base_level = 1,
+ bool level_compaction_dynamic_level_bytes = false);
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/db_options.cc b/src/rocksdb/options/db_options.cc
new file mode 100644
index 000000000..0dcc8ef79
--- /dev/null
+++ b/src/rocksdb/options/db_options.cc
@@ -0,0 +1,333 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "options/db_options.h"
+
+#include <cinttypes>
+
+#include "db/version_edit.h"
+#include "logging/logging.h"
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/env.h"
+#include "rocksdb/file_system.h"
+#include "rocksdb/sst_file_manager.h"
+#include "rocksdb/wal_filter.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {}
+
+ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
+ : create_if_missing(options.create_if_missing),
+ create_missing_column_families(options.create_missing_column_families),
+ error_if_exists(options.error_if_exists),
+ paranoid_checks(options.paranoid_checks),
+ env(options.env),
+ fs(options.file_system),
+ rate_limiter(options.rate_limiter),
+ sst_file_manager(options.sst_file_manager),
+ info_log(options.info_log),
+ info_log_level(options.info_log_level),
+ max_file_opening_threads(options.max_file_opening_threads),
+ statistics(options.statistics),
+ use_fsync(options.use_fsync),
+ db_paths(options.db_paths),
+ db_log_dir(options.db_log_dir),
+ wal_dir(options.wal_dir),
+ max_subcompactions(options.max_subcompactions),
+ max_background_flushes(options.max_background_flushes),
+ max_log_file_size(options.max_log_file_size),
+ log_file_time_to_roll(options.log_file_time_to_roll),
+ keep_log_file_num(options.keep_log_file_num),
+ recycle_log_file_num(options.recycle_log_file_num),
+ max_manifest_file_size(options.max_manifest_file_size),
+ table_cache_numshardbits(options.table_cache_numshardbits),
+ wal_ttl_seconds(options.WAL_ttl_seconds),
+ wal_size_limit_mb(options.WAL_size_limit_MB),
+ max_write_batch_group_size_bytes(
+ options.max_write_batch_group_size_bytes),
+ manifest_preallocation_size(options.manifest_preallocation_size),
+ allow_mmap_reads(options.allow_mmap_reads),
+ allow_mmap_writes(options.allow_mmap_writes),
+ use_direct_reads(options.use_direct_reads),
+ use_direct_io_for_flush_and_compaction(
+ options.use_direct_io_for_flush_and_compaction),
+ allow_fallocate(options.allow_fallocate),
+ is_fd_close_on_exec(options.is_fd_close_on_exec),
+ advise_random_on_open(options.advise_random_on_open),
+ db_write_buffer_size(options.db_write_buffer_size),
+ write_buffer_manager(options.write_buffer_manager),
+ access_hint_on_compaction_start(options.access_hint_on_compaction_start),
+ new_table_reader_for_compaction_inputs(
+ options.new_table_reader_for_compaction_inputs),
+ random_access_max_buffer_size(options.random_access_max_buffer_size),
+ use_adaptive_mutex(options.use_adaptive_mutex),
+ listeners(options.listeners),
+ enable_thread_tracking(options.enable_thread_tracking),
+ enable_pipelined_write(options.enable_pipelined_write),
+ unordered_write(options.unordered_write),
+ allow_concurrent_memtable_write(options.allow_concurrent_memtable_write),
+ enable_write_thread_adaptive_yield(
+ options.enable_write_thread_adaptive_yield),
+ write_thread_max_yield_usec(options.write_thread_max_yield_usec),
+ write_thread_slow_yield_usec(options.write_thread_slow_yield_usec),
+ skip_stats_update_on_db_open(options.skip_stats_update_on_db_open),
+ skip_checking_sst_file_sizes_on_db_open(
+ options.skip_checking_sst_file_sizes_on_db_open),
+ wal_recovery_mode(options.wal_recovery_mode),
+ allow_2pc(options.allow_2pc),
+ row_cache(options.row_cache),
+#ifndef ROCKSDB_LITE
+ wal_filter(options.wal_filter),
+#endif // ROCKSDB_LITE
+ fail_if_options_file_error(options.fail_if_options_file_error),
+ dump_malloc_stats(options.dump_malloc_stats),
+ avoid_flush_during_recovery(options.avoid_flush_during_recovery),
+ allow_ingest_behind(options.allow_ingest_behind),
+ preserve_deletes(options.preserve_deletes),
+ two_write_queues(options.two_write_queues),
+ manual_wal_flush(options.manual_wal_flush),
+ atomic_flush(options.atomic_flush),
+ avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io),
+ persist_stats_to_disk(options.persist_stats_to_disk),
+ write_dbid_to_manifest(options.write_dbid_to_manifest),
+ log_readahead_size(options.log_readahead_size),
+ sst_file_checksum_func(options.sst_file_checksum_func) {
+}
+
+void ImmutableDBOptions::Dump(Logger* log) const {
+ ROCKS_LOG_HEADER(log, " Options.error_if_exists: %d",
+ error_if_exists);
+ ROCKS_LOG_HEADER(log, " Options.create_if_missing: %d",
+ create_if_missing);
+ ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d",
+ paranoid_checks);
+ ROCKS_LOG_HEADER(log, " Options.env: %p",
+ env);
+ ROCKS_LOG_HEADER(log, " Options.fs: %s",
+ fs->Name());
+ ROCKS_LOG_HEADER(log, " Options.info_log: %p",
+ info_log.get());
+ ROCKS_LOG_HEADER(log, " Options.max_file_opening_threads: %d",
+ max_file_opening_threads);
+ ROCKS_LOG_HEADER(log, " Options.statistics: %p",
+ statistics.get());
+ ROCKS_LOG_HEADER(log, " Options.use_fsync: %d",
+ use_fsync);
+ ROCKS_LOG_HEADER(
+ log, " Options.max_log_file_size: %" ROCKSDB_PRIszt,
+ max_log_file_size);
+ ROCKS_LOG_HEADER(log,
+ " Options.max_manifest_file_size: %" PRIu64,
+ max_manifest_file_size);
+ ROCKS_LOG_HEADER(
+ log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt,
+ log_file_time_to_roll);
+ ROCKS_LOG_HEADER(
+ log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt,
+ keep_log_file_num);
+ ROCKS_LOG_HEADER(
+ log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt,
+ recycle_log_file_num);
+ ROCKS_LOG_HEADER(log, " Options.allow_fallocate: %d",
+ allow_fallocate);
+ ROCKS_LOG_HEADER(log, " Options.allow_mmap_reads: %d",
+ allow_mmap_reads);
+ ROCKS_LOG_HEADER(log, " Options.allow_mmap_writes: %d",
+ allow_mmap_writes);
+ ROCKS_LOG_HEADER(log, " Options.use_direct_reads: %d",
+ use_direct_reads);
+ ROCKS_LOG_HEADER(log,
+ " "
+ "Options.use_direct_io_for_flush_and_compaction: %d",
+ use_direct_io_for_flush_and_compaction);
+ ROCKS_LOG_HEADER(log, " Options.create_missing_column_families: %d",
+ create_missing_column_families);
+ ROCKS_LOG_HEADER(log, " Options.db_log_dir: %s",
+ db_log_dir.c_str());
+ ROCKS_LOG_HEADER(log, " Options.wal_dir: %s",
+ wal_dir.c_str());
+ ROCKS_LOG_HEADER(log, " Options.table_cache_numshardbits: %d",
+ table_cache_numshardbits);
+ ROCKS_LOG_HEADER(log,
+ " Options.max_subcompactions: %" PRIu32,
+ max_subcompactions);
+ ROCKS_LOG_HEADER(log, " Options.max_background_flushes: %d",
+ max_background_flushes);
+ ROCKS_LOG_HEADER(log,
+ " Options.WAL_ttl_seconds: %" PRIu64,
+ wal_ttl_seconds);
+ ROCKS_LOG_HEADER(log,
+ " Options.WAL_size_limit_MB: %" PRIu64,
+ wal_size_limit_mb);
+ ROCKS_LOG_HEADER(log,
+ " "
+ "Options.max_write_batch_group_size_bytes: %" PRIu64,
+ max_write_batch_group_size_bytes);
+ ROCKS_LOG_HEADER(
+ log, " Options.manifest_preallocation_size: %" ROCKSDB_PRIszt,
+ manifest_preallocation_size);
+ ROCKS_LOG_HEADER(log, " Options.is_fd_close_on_exec: %d",
+ is_fd_close_on_exec);
+ ROCKS_LOG_HEADER(log, " Options.advise_random_on_open: %d",
+ advise_random_on_open);
+ ROCKS_LOG_HEADER(
+ log, " Options.db_write_buffer_size: %" ROCKSDB_PRIszt,
+ db_write_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.write_buffer_manager: %p",
+ write_buffer_manager.get());
+ ROCKS_LOG_HEADER(log, " Options.access_hint_on_compaction_start: %d",
+ static_cast<int>(access_hint_on_compaction_start));
+ ROCKS_LOG_HEADER(log, " Options.new_table_reader_for_compaction_inputs: %d",
+ new_table_reader_for_compaction_inputs);
+ ROCKS_LOG_HEADER(
+ log, " Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt,
+ random_access_max_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.use_adaptive_mutex: %d",
+ use_adaptive_mutex);
+ ROCKS_LOG_HEADER(log, " Options.rate_limiter: %p",
+ rate_limiter.get());
+ Header(
+ log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64,
+ sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0);
+ ROCKS_LOG_HEADER(log, " Options.wal_recovery_mode: %d",
+ static_cast<int>(wal_recovery_mode));
+ ROCKS_LOG_HEADER(log, " Options.enable_thread_tracking: %d",
+ enable_thread_tracking);
+ ROCKS_LOG_HEADER(log, " Options.enable_pipelined_write: %d",
+ enable_pipelined_write);
+ ROCKS_LOG_HEADER(log, " Options.unordered_write: %d",
+ unordered_write);
+ ROCKS_LOG_HEADER(log, " Options.allow_concurrent_memtable_write: %d",
+ allow_concurrent_memtable_write);
+ ROCKS_LOG_HEADER(log, " Options.enable_write_thread_adaptive_yield: %d",
+ enable_write_thread_adaptive_yield);
+ ROCKS_LOG_HEADER(log,
+ " Options.write_thread_max_yield_usec: %" PRIu64,
+ write_thread_max_yield_usec);
+ ROCKS_LOG_HEADER(log,
+ " Options.write_thread_slow_yield_usec: %" PRIu64,
+ write_thread_slow_yield_usec);
+ if (row_cache) {
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.row_cache: %" ROCKSDB_PRIszt,
+ row_cache->GetCapacity());
+ } else {
+ ROCKS_LOG_HEADER(log,
+ " Options.row_cache: None");
+ }
+#ifndef ROCKSDB_LITE
+ ROCKS_LOG_HEADER(log, " Options.wal_filter: %s",
+ wal_filter ? wal_filter->Name() : "None");
+#endif // ROCKDB_LITE
+
+ ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_recovery: %d",
+ avoid_flush_during_recovery);
+ ROCKS_LOG_HEADER(log, " Options.allow_ingest_behind: %d",
+ allow_ingest_behind);
+ ROCKS_LOG_HEADER(log, " Options.preserve_deletes: %d",
+ preserve_deletes);
+ ROCKS_LOG_HEADER(log, " Options.two_write_queues: %d",
+ two_write_queues);
+ ROCKS_LOG_HEADER(log, " Options.manual_wal_flush: %d",
+ manual_wal_flush);
+ ROCKS_LOG_HEADER(log, " Options.atomic_flush: %d", atomic_flush);
+ ROCKS_LOG_HEADER(log,
+ " Options.avoid_unnecessary_blocking_io: %d",
+ avoid_unnecessary_blocking_io);
+ ROCKS_LOG_HEADER(log, " Options.persist_stats_to_disk: %u",
+ persist_stats_to_disk);
+ ROCKS_LOG_HEADER(log, " Options.write_dbid_to_manifest: %d",
+ write_dbid_to_manifest);
+ ROCKS_LOG_HEADER(
+ log, " Options.log_readahead_size: %" ROCKSDB_PRIszt,
+ log_readahead_size);
+ ROCKS_LOG_HEADER(log, " Options.sst_file_checksum_func: %s",
+ sst_file_checksum_func
+ ? sst_file_checksum_func->Name()
+ : kUnknownFileChecksumFuncName.c_str());
+}
+
+MutableDBOptions::MutableDBOptions()
+ : max_background_jobs(2),
+ base_background_compactions(-1),
+ max_background_compactions(-1),
+ avoid_flush_during_shutdown(false),
+ writable_file_max_buffer_size(1024 * 1024),
+ delayed_write_rate(2 * 1024U * 1024U),
+ max_total_wal_size(0),
+ delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000),
+ stats_dump_period_sec(600),
+ stats_persist_period_sec(600),
+ stats_history_buffer_size(1024 * 1024),
+ max_open_files(-1),
+ bytes_per_sync(0),
+ wal_bytes_per_sync(0),
+ strict_bytes_per_sync(false),
+ compaction_readahead_size(0) {}
+
+MutableDBOptions::MutableDBOptions(const DBOptions& options)
+ : max_background_jobs(options.max_background_jobs),
+ base_background_compactions(options.base_background_compactions),
+ max_background_compactions(options.max_background_compactions),
+ avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
+ writable_file_max_buffer_size(options.writable_file_max_buffer_size),
+ delayed_write_rate(options.delayed_write_rate),
+ max_total_wal_size(options.max_total_wal_size),
+ delete_obsolete_files_period_micros(
+ options.delete_obsolete_files_period_micros),
+ stats_dump_period_sec(options.stats_dump_period_sec),
+ stats_persist_period_sec(options.stats_persist_period_sec),
+ stats_history_buffer_size(options.stats_history_buffer_size),
+ max_open_files(options.max_open_files),
+ bytes_per_sync(options.bytes_per_sync),
+ wal_bytes_per_sync(options.wal_bytes_per_sync),
+ strict_bytes_per_sync(options.strict_bytes_per_sync),
+ compaction_readahead_size(options.compaction_readahead_size) {}
+
+void MutableDBOptions::Dump(Logger* log) const {
+ ROCKS_LOG_HEADER(log, " Options.max_background_jobs: %d",
+ max_background_jobs);
+ ROCKS_LOG_HEADER(log, " Options.max_background_compactions: %d",
+ max_background_compactions);
+ ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_shutdown: %d",
+ avoid_flush_during_shutdown);
+ ROCKS_LOG_HEADER(
+ log, " Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt,
+ writable_file_max_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.delayed_write_rate : %" PRIu64,
+ delayed_write_rate);
+ ROCKS_LOG_HEADER(log, " Options.max_total_wal_size: %" PRIu64,
+ max_total_wal_size);
+ ROCKS_LOG_HEADER(
+ log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
+ delete_obsolete_files_period_micros);
+ ROCKS_LOG_HEADER(log, " Options.stats_dump_period_sec: %u",
+ stats_dump_period_sec);
+ ROCKS_LOG_HEADER(log, " Options.stats_persist_period_sec: %d",
+ stats_persist_period_sec);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.stats_history_buffer_size: %" ROCKSDB_PRIszt,
+ stats_history_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.max_open_files: %d",
+ max_open_files);
+ ROCKS_LOG_HEADER(log,
+ " Options.bytes_per_sync: %" PRIu64,
+ bytes_per_sync);
+ ROCKS_LOG_HEADER(log,
+ " Options.wal_bytes_per_sync: %" PRIu64,
+ wal_bytes_per_sync);
+ ROCKS_LOG_HEADER(log,
+ " Options.strict_bytes_per_sync: %d",
+ strict_bytes_per_sync);
+ ROCKS_LOG_HEADER(log,
+ " Options.compaction_readahead_size: %" ROCKSDB_PRIszt,
+ compaction_readahead_size);
+}
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/db_options.h b/src/rocksdb/options/db_options.h
new file mode 100644
index 000000000..cba298a21
--- /dev/null
+++ b/src/rocksdb/options/db_options.h
@@ -0,0 +1,118 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "rocksdb/options.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+struct ImmutableDBOptions {
+ ImmutableDBOptions();
+ explicit ImmutableDBOptions(const DBOptions& options);
+
+ void Dump(Logger* log) const;
+
+ bool create_if_missing;
+ bool create_missing_column_families;
+ bool error_if_exists;
+ bool paranoid_checks;
+ Env* env;
+ std::shared_ptr<FileSystem> fs;
+ std::shared_ptr<RateLimiter> rate_limiter;
+ std::shared_ptr<SstFileManager> sst_file_manager;
+ std::shared_ptr<Logger> info_log;
+ InfoLogLevel info_log_level;
+ int max_file_opening_threads;
+ std::shared_ptr<Statistics> statistics;
+ bool use_fsync;
+ std::vector<DbPath> db_paths;
+ std::string db_log_dir;
+ std::string wal_dir;
+ uint32_t max_subcompactions;
+ int max_background_flushes;
+ size_t max_log_file_size;
+ size_t log_file_time_to_roll;
+ size_t keep_log_file_num;
+ size_t recycle_log_file_num;
+ uint64_t max_manifest_file_size;
+ int table_cache_numshardbits;
+ uint64_t wal_ttl_seconds;
+ uint64_t wal_size_limit_mb;
+ uint64_t max_write_batch_group_size_bytes;
+ size_t manifest_preallocation_size;
+ bool allow_mmap_reads;
+ bool allow_mmap_writes;
+ bool use_direct_reads;
+ bool use_direct_io_for_flush_and_compaction;
+ bool allow_fallocate;
+ bool is_fd_close_on_exec;
+ bool advise_random_on_open;
+ size_t db_write_buffer_size;
+ std::shared_ptr<WriteBufferManager> write_buffer_manager;
+ DBOptions::AccessHint access_hint_on_compaction_start;
+ bool new_table_reader_for_compaction_inputs;
+ size_t random_access_max_buffer_size;
+ bool use_adaptive_mutex;
+ std::vector<std::shared_ptr<EventListener>> listeners;
+ bool enable_thread_tracking;
+ bool enable_pipelined_write;
+ bool unordered_write;
+ bool allow_concurrent_memtable_write;
+ bool enable_write_thread_adaptive_yield;
+ uint64_t write_thread_max_yield_usec;
+ uint64_t write_thread_slow_yield_usec;
+ bool skip_stats_update_on_db_open;
+ bool skip_checking_sst_file_sizes_on_db_open;
+ WALRecoveryMode wal_recovery_mode;
+ bool allow_2pc;
+ std::shared_ptr<Cache> row_cache;
+#ifndef ROCKSDB_LITE
+ WalFilter* wal_filter;
+#endif // ROCKSDB_LITE
+ bool fail_if_options_file_error;
+ bool dump_malloc_stats;
+ bool avoid_flush_during_recovery;
+ bool allow_ingest_behind;
+ bool preserve_deletes;
+ bool two_write_queues;
+ bool manual_wal_flush;
+ bool atomic_flush;
+ bool avoid_unnecessary_blocking_io;
+ bool persist_stats_to_disk;
+ bool write_dbid_to_manifest;
+ size_t log_readahead_size;
+ std::shared_ptr<FileChecksumFunc> sst_file_checksum_func;
+};
+
+struct MutableDBOptions {
+ MutableDBOptions();
+ explicit MutableDBOptions(const MutableDBOptions& options) = default;
+ explicit MutableDBOptions(const DBOptions& options);
+
+ void Dump(Logger* log) const;
+
+ int max_background_jobs;
+ int base_background_compactions;
+ int max_background_compactions;
+ bool avoid_flush_during_shutdown;
+ size_t writable_file_max_buffer_size;
+ uint64_t delayed_write_rate;
+ uint64_t max_total_wal_size;
+ uint64_t delete_obsolete_files_period_micros;
+ unsigned int stats_dump_period_sec;
+ unsigned int stats_persist_period_sec;
+ size_t stats_history_buffer_size;
+ int max_open_files;
+ uint64_t bytes_per_sync;
+ uint64_t wal_bytes_per_sync;
+ bool strict_bytes_per_sync;
+ size_t compaction_readahead_size;
+};
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options.cc b/src/rocksdb/options/options.cc
new file mode 100644
index 000000000..fe676597e
--- /dev/null
+++ b/src/rocksdb/options/options.cc
@@ -0,0 +1,623 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "rocksdb/options.h"
+
+#include <cinttypes>
+#include <limits>
+
+#include "monitoring/statistics.h"
+#include "options/db_options.h"
+#include "options/options_helper.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/sst_file_manager.h"
+#include "rocksdb/table.h"
+#include "rocksdb/table_properties.h"
+#include "rocksdb/wal_filter.h"
+#include "table/block_based/block_based_table_factory.h"
+#include "util/compression.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions() {
+ assert(memtable_factory.get() != nullptr);
+}
+
+AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options)
+ : max_write_buffer_number(options.max_write_buffer_number),
+ min_write_buffer_number_to_merge(
+ options.min_write_buffer_number_to_merge),
+ max_write_buffer_number_to_maintain(
+ options.max_write_buffer_number_to_maintain),
+ max_write_buffer_size_to_maintain(
+ options.max_write_buffer_size_to_maintain),
+ inplace_update_support(options.inplace_update_support),
+ inplace_update_num_locks(options.inplace_update_num_locks),
+ inplace_callback(options.inplace_callback),
+ memtable_prefix_bloom_size_ratio(
+ options.memtable_prefix_bloom_size_ratio),
+ memtable_whole_key_filtering(options.memtable_whole_key_filtering),
+ memtable_huge_page_size(options.memtable_huge_page_size),
+ memtable_insert_with_hint_prefix_extractor(
+ options.memtable_insert_with_hint_prefix_extractor),
+ bloom_locality(options.bloom_locality),
+ arena_block_size(options.arena_block_size),
+ compression_per_level(options.compression_per_level),
+ num_levels(options.num_levels),
+ level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
+ level0_stop_writes_trigger(options.level0_stop_writes_trigger),
+ target_file_size_base(options.target_file_size_base),
+ target_file_size_multiplier(options.target_file_size_multiplier),
+ level_compaction_dynamic_level_bytes(
+ options.level_compaction_dynamic_level_bytes),
+ max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
+ max_bytes_for_level_multiplier_additional(
+ options.max_bytes_for_level_multiplier_additional),
+ max_compaction_bytes(options.max_compaction_bytes),
+ soft_pending_compaction_bytes_limit(
+ options.soft_pending_compaction_bytes_limit),
+ hard_pending_compaction_bytes_limit(
+ options.hard_pending_compaction_bytes_limit),
+ compaction_style(options.compaction_style),
+ compaction_pri(options.compaction_pri),
+ compaction_options_universal(options.compaction_options_universal),
+ compaction_options_fifo(options.compaction_options_fifo),
+ max_sequential_skip_in_iterations(
+ options.max_sequential_skip_in_iterations),
+ memtable_factory(options.memtable_factory),
+ table_properties_collector_factories(
+ options.table_properties_collector_factories),
+ max_successive_merges(options.max_successive_merges),
+ optimize_filters_for_hits(options.optimize_filters_for_hits),
+ paranoid_file_checks(options.paranoid_file_checks),
+ force_consistency_checks(options.force_consistency_checks),
+ report_bg_io_stats(options.report_bg_io_stats),
+ ttl(options.ttl),
+ periodic_compaction_seconds(options.periodic_compaction_seconds),
+ sample_for_compression(options.sample_for_compression) {
+ assert(memtable_factory.get() != nullptr);
+ if (max_bytes_for_level_multiplier_additional.size() <
+ static_cast<unsigned int>(num_levels)) {
+ max_bytes_for_level_multiplier_additional.resize(num_levels, 1);
+ }
+}
+
+ColumnFamilyOptions::ColumnFamilyOptions()
+ : compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
+ table_factory(
+ std::shared_ptr<TableFactory>(new BlockBasedTableFactory())) {}
+
+ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
+ : ColumnFamilyOptions(*static_cast<const ColumnFamilyOptions*>(&options)) {}
+
+DBOptions::DBOptions() {}
+DBOptions::DBOptions(const Options& options)
+ : DBOptions(*static_cast<const DBOptions*>(&options)) {}
+
+void DBOptions::Dump(Logger* log) const {
+ ImmutableDBOptions(*this).Dump(log);
+ MutableDBOptions(*this).Dump(log);
+} // DBOptions::Dump
+
+void ColumnFamilyOptions::Dump(Logger* log) const {
+ ROCKS_LOG_HEADER(log, " Options.comparator: %s",
+ comparator->Name());
+ ROCKS_LOG_HEADER(log, " Options.merge_operator: %s",
+ merge_operator ? merge_operator->Name() : "None");
+ ROCKS_LOG_HEADER(log, " Options.compaction_filter: %s",
+ compaction_filter ? compaction_filter->Name() : "None");
+ ROCKS_LOG_HEADER(
+ log, " Options.compaction_filter_factory: %s",
+ compaction_filter_factory ? compaction_filter_factory->Name() : "None");
+ ROCKS_LOG_HEADER(log, " Options.memtable_factory: %s",
+ memtable_factory->Name());
+ ROCKS_LOG_HEADER(log, " Options.table_factory: %s",
+ table_factory->Name());
+ ROCKS_LOG_HEADER(log, " table_factory options: %s",
+ table_factory->GetPrintableTableOptions().c_str());
+ ROCKS_LOG_HEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt,
+ write_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number: %d",
+ max_write_buffer_number);
+ if (!compression_per_level.empty()) {
+ for (unsigned int i = 0; i < compression_per_level.size(); i++) {
+ ROCKS_LOG_HEADER(
+ log, " Options.compression[%d]: %s", i,
+ CompressionTypeToString(compression_per_level[i]).c_str());
+ }
+ } else {
+ ROCKS_LOG_HEADER(log, " Options.compression: %s",
+ CompressionTypeToString(compression).c_str());
+ }
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression: %s",
+ bottommost_compression == kDisableCompressionOption
+ ? "Disabled"
+ : CompressionTypeToString(bottommost_compression).c_str());
+ ROCKS_LOG_HEADER(
+ log, " Options.prefix_extractor: %s",
+ prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
+ ROCKS_LOG_HEADER(log,
+ " Options.memtable_insert_with_hint_prefix_extractor: %s",
+ memtable_insert_with_hint_prefix_extractor == nullptr
+ ? "nullptr"
+ : memtable_insert_with_hint_prefix_extractor->Name());
+ ROCKS_LOG_HEADER(log, " Options.num_levels: %d", num_levels);
+ ROCKS_LOG_HEADER(log, " Options.min_write_buffer_number_to_merge: %d",
+ min_write_buffer_number_to_merge);
+ ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number_to_maintain: %d",
+ max_write_buffer_number_to_maintain);
+ ROCKS_LOG_HEADER(log,
+ " Options.max_write_buffer_size_to_maintain: %" PRIu64,
+ max_write_buffer_size_to_maintain);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.window_bits: %d",
+ bottommost_compression_opts.window_bits);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.level: %d",
+ bottommost_compression_opts.level);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.strategy: %d",
+ bottommost_compression_opts.strategy);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.bottommost_compression_opts.max_dict_bytes: "
+ "%" PRIu32,
+ bottommost_compression_opts.max_dict_bytes);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.bottommost_compression_opts.zstd_max_train_bytes: "
+ "%" PRIu32,
+ bottommost_compression_opts.zstd_max_train_bytes);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.enabled: %s",
+ bottommost_compression_opts.enabled ? "true" : "false");
+ ROCKS_LOG_HEADER(log, " Options.compression_opts.window_bits: %d",
+ compression_opts.window_bits);
+ ROCKS_LOG_HEADER(log, " Options.compression_opts.level: %d",
+ compression_opts.level);
+ ROCKS_LOG_HEADER(log, " Options.compression_opts.strategy: %d",
+ compression_opts.strategy);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.compression_opts.max_dict_bytes: %" PRIu32,
+ compression_opts.max_dict_bytes);
+ ROCKS_LOG_HEADER(log,
+ " Options.compression_opts.zstd_max_train_bytes: "
+ "%" PRIu32,
+ compression_opts.zstd_max_train_bytes);
+ ROCKS_LOG_HEADER(log,
+ " Options.compression_opts.enabled: %s",
+ compression_opts.enabled ? "true" : "false");
+ ROCKS_LOG_HEADER(log, " Options.level0_file_num_compaction_trigger: %d",
+ level0_file_num_compaction_trigger);
+ ROCKS_LOG_HEADER(log, " Options.level0_slowdown_writes_trigger: %d",
+ level0_slowdown_writes_trigger);
+ ROCKS_LOG_HEADER(log, " Options.level0_stop_writes_trigger: %d",
+ level0_stop_writes_trigger);
+ ROCKS_LOG_HEADER(
+ log, " Options.target_file_size_base: %" PRIu64,
+ target_file_size_base);
+ ROCKS_LOG_HEADER(log, " Options.target_file_size_multiplier: %d",
+ target_file_size_multiplier);
+ ROCKS_LOG_HEADER(
+ log, " Options.max_bytes_for_level_base: %" PRIu64,
+ max_bytes_for_level_base);
+ ROCKS_LOG_HEADER(log, "Options.level_compaction_dynamic_level_bytes: %d",
+ level_compaction_dynamic_level_bytes);
+ ROCKS_LOG_HEADER(log, " Options.max_bytes_for_level_multiplier: %f",
+ max_bytes_for_level_multiplier);
+ for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size();
+ i++) {
+ ROCKS_LOG_HEADER(
+ log, "Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt
+ "]: %d",
+ i, max_bytes_for_level_multiplier_additional[i]);
+ }
+ ROCKS_LOG_HEADER(
+ log, " Options.max_sequential_skip_in_iterations: %" PRIu64,
+ max_sequential_skip_in_iterations);
+ ROCKS_LOG_HEADER(
+ log, " Options.max_compaction_bytes: %" PRIu64,
+ max_compaction_bytes);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.arena_block_size: %" ROCKSDB_PRIszt,
+ arena_block_size);
+ ROCKS_LOG_HEADER(log,
+ " Options.soft_pending_compaction_bytes_limit: %" PRIu64,
+ soft_pending_compaction_bytes_limit);
+ ROCKS_LOG_HEADER(log,
+ " Options.hard_pending_compaction_bytes_limit: %" PRIu64,
+ hard_pending_compaction_bytes_limit);
+ ROCKS_LOG_HEADER(log, " Options.rate_limit_delay_max_milliseconds: %u",
+ rate_limit_delay_max_milliseconds);
+ ROCKS_LOG_HEADER(log, " Options.disable_auto_compactions: %d",
+ disable_auto_compactions);
+
+ const auto& it_compaction_style =
+ compaction_style_to_string.find(compaction_style);
+ std::string str_compaction_style;
+ if (it_compaction_style == compaction_style_to_string.end()) {
+ assert(false);
+ str_compaction_style = "unknown_" + std::to_string(compaction_style);
+ } else {
+ str_compaction_style = it_compaction_style->second;
+ }
+ ROCKS_LOG_HEADER(log,
+ " Options.compaction_style: %s",
+ str_compaction_style.c_str());
+
+ const auto& it_compaction_pri =
+ compaction_pri_to_string.find(compaction_pri);
+ std::string str_compaction_pri;
+ if (it_compaction_pri == compaction_pri_to_string.end()) {
+ assert(false);
+ str_compaction_pri = "unknown_" + std::to_string(compaction_pri);
+ } else {
+ str_compaction_pri = it_compaction_pri->second;
+ }
+ ROCKS_LOG_HEADER(log,
+ " Options.compaction_pri: %s",
+ str_compaction_pri.c_str());
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.size_ratio: %u",
+ compaction_options_universal.size_ratio);
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.min_merge_width: %u",
+ compaction_options_universal.min_merge_width);
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.max_merge_width: %u",
+ compaction_options_universal.max_merge_width);
+ ROCKS_LOG_HEADER(
+ log,
+ "Options.compaction_options_universal."
+ "max_size_amplification_percent: %u",
+ compaction_options_universal.max_size_amplification_percent);
+ ROCKS_LOG_HEADER(
+ log,
+ "Options.compaction_options_universal.compression_size_percent: %d",
+ compaction_options_universal.compression_size_percent);
+ const auto& it_compaction_stop_style = compaction_stop_style_to_string.find(
+ compaction_options_universal.stop_style);
+ std::string str_compaction_stop_style;
+ if (it_compaction_stop_style == compaction_stop_style_to_string.end()) {
+ assert(false);
+ str_compaction_stop_style =
+ "unknown_" + std::to_string(compaction_options_universal.stop_style);
+ } else {
+ str_compaction_stop_style = it_compaction_stop_style->second;
+ }
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.stop_style: %s",
+ str_compaction_stop_style.c_str());
+ ROCKS_LOG_HEADER(
+ log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64,
+ compaction_options_fifo.max_table_files_size);
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_fifo.allow_compaction: %d",
+ compaction_options_fifo.allow_compaction);
+ std::string collector_names;
+ for (const auto& collector_factory : table_properties_collector_factories) {
+ collector_names.append(collector_factory->Name());
+ collector_names.append("; ");
+ }
+ ROCKS_LOG_HEADER(
+ log, " Options.table_properties_collectors: %s",
+ collector_names.c_str());
+ ROCKS_LOG_HEADER(log,
+ " Options.inplace_update_support: %d",
+ inplace_update_support);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.inplace_update_num_locks: %" ROCKSDB_PRIszt,
+ inplace_update_num_locks);
+ // TODO: easier config for bloom (maybe based on avg key/value size)
+ ROCKS_LOG_HEADER(
+ log, " Options.memtable_prefix_bloom_size_ratio: %f",
+ memtable_prefix_bloom_size_ratio);
+ ROCKS_LOG_HEADER(log,
+ " Options.memtable_whole_key_filtering: %d",
+ memtable_whole_key_filtering);
+
+ ROCKS_LOG_HEADER(log, " Options.memtable_huge_page_size: %" ROCKSDB_PRIszt,
+ memtable_huge_page_size);
+ ROCKS_LOG_HEADER(log,
+ " Options.bloom_locality: %d",
+ bloom_locality);
+
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.max_successive_merges: %" ROCKSDB_PRIszt,
+ max_successive_merges);
+ ROCKS_LOG_HEADER(log,
+ " Options.optimize_filters_for_hits: %d",
+ optimize_filters_for_hits);
+ ROCKS_LOG_HEADER(log, " Options.paranoid_file_checks: %d",
+ paranoid_file_checks);
+ ROCKS_LOG_HEADER(log, " Options.force_consistency_checks: %d",
+ force_consistency_checks);
+ ROCKS_LOG_HEADER(log, " Options.report_bg_io_stats: %d",
+ report_bg_io_stats);
+ ROCKS_LOG_HEADER(log, " Options.ttl: %" PRIu64,
+ ttl);
+ ROCKS_LOG_HEADER(log,
+ " Options.periodic_compaction_seconds: %" PRIu64,
+ periodic_compaction_seconds);
+} // ColumnFamilyOptions::Dump
+
+void Options::Dump(Logger* log) const {
+ DBOptions::Dump(log);
+ ColumnFamilyOptions::Dump(log);
+} // Options::Dump
+
+void Options::DumpCFOptions(Logger* log) const {
+ ColumnFamilyOptions::Dump(log);
+} // Options::DumpCFOptions
+
+//
+// The goal of this method is to create a configuration that
+// allows an application to write all files into L0 and
+// then do a single compaction to output all files into L1.
+Options*
+Options::PrepareForBulkLoad()
+{
+ // never slowdown ingest.
+ level0_file_num_compaction_trigger = (1<<30);
+ level0_slowdown_writes_trigger = (1<<30);
+ level0_stop_writes_trigger = (1<<30);
+ soft_pending_compaction_bytes_limit = 0;
+ hard_pending_compaction_bytes_limit = 0;
+
+ // no auto compactions please. The application should issue a
+ // manual compaction after all data is loaded into L0.
+ disable_auto_compactions = true;
+ // A manual compaction run should pick all files in L0 in
+ // a single compaction run.
+ max_compaction_bytes = (static_cast<uint64_t>(1) << 60);
+
+ // It is better to have only 2 levels, otherwise a manual
+ // compaction would compact at every possible level, thereby
+ // increasing the total time needed for compactions.
+ num_levels = 2;
+
+ // Need to allow more write buffers to allow more parallism
+ // of flushes.
+ max_write_buffer_number = 6;
+ min_write_buffer_number_to_merge = 1;
+
+ // When compaction is disabled, more parallel flush threads can
+ // help with write throughput.
+ max_background_flushes = 4;
+
+ // Prevent a memtable flush to automatically promote files
+ // to L1. This is helpful so that all files that are
+ // input to the manual compaction are all at L0.
+ max_background_compactions = 2;
+
+ // The compaction would create large files in L1.
+ target_file_size_base = 256 * 1024 * 1024;
+ return this;
+}
+
+Options* Options::OptimizeForSmallDb() {
+ // 16MB block cache
+ std::shared_ptr<Cache> cache = NewLRUCache(16 << 20);
+
+ ColumnFamilyOptions::OptimizeForSmallDb(&cache);
+ DBOptions::OptimizeForSmallDb(&cache);
+ return this;
+}
+
+Options* Options::OldDefaults(int rocksdb_major_version,
+ int rocksdb_minor_version) {
+ ColumnFamilyOptions::OldDefaults(rocksdb_major_version,
+ rocksdb_minor_version);
+ DBOptions::OldDefaults(rocksdb_major_version, rocksdb_minor_version);
+ return this;
+}
+
+DBOptions* DBOptions::OldDefaults(int rocksdb_major_version,
+ int rocksdb_minor_version) {
+ if (rocksdb_major_version < 4 ||
+ (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) {
+ max_file_opening_threads = 1;
+ table_cache_numshardbits = 4;
+ }
+ if (rocksdb_major_version < 5 ||
+ (rocksdb_major_version == 5 && rocksdb_minor_version < 2)) {
+ delayed_write_rate = 2 * 1024U * 1024U;
+ } else if (rocksdb_major_version < 5 ||
+ (rocksdb_major_version == 5 && rocksdb_minor_version < 6)) {
+ delayed_write_rate = 16 * 1024U * 1024U;
+ }
+ max_open_files = 5000;
+ wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OldDefaults(
+ int rocksdb_major_version, int rocksdb_minor_version) {
+ if (rocksdb_major_version < 5 ||
+ (rocksdb_major_version == 5 && rocksdb_minor_version <= 18)) {
+ compaction_pri = CompactionPri::kByCompensatedSize;
+ }
+ if (rocksdb_major_version < 4 ||
+ (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) {
+ write_buffer_size = 4 << 20;
+ target_file_size_base = 2 * 1048576;
+ max_bytes_for_level_base = 10 * 1048576;
+ soft_pending_compaction_bytes_limit = 0;
+ hard_pending_compaction_bytes_limit = 0;
+ }
+ if (rocksdb_major_version < 5) {
+ level0_stop_writes_trigger = 24;
+ } else if (rocksdb_major_version == 5 && rocksdb_minor_version < 2) {
+ level0_stop_writes_trigger = 30;
+ }
+
+ return this;
+}
+
+// Optimization functions
+DBOptions* DBOptions::OptimizeForSmallDb(std::shared_ptr<Cache>* cache) {
+ max_file_opening_threads = 1;
+ max_open_files = 5000;
+
+ // Cost memtable to block cache too.
+ std::shared_ptr<ROCKSDB_NAMESPACE::WriteBufferManager> wbm =
+ std::make_shared<ROCKSDB_NAMESPACE::WriteBufferManager>(
+ 0, (cache != nullptr) ? *cache : std::shared_ptr<Cache>());
+ write_buffer_manager = wbm;
+
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForSmallDb(
+ std::shared_ptr<Cache>* cache) {
+ write_buffer_size = 2 << 20;
+ target_file_size_base = 2 * 1048576;
+ max_bytes_for_level_base = 10 * 1048576;
+ soft_pending_compaction_bytes_limit = 256 * 1048576;
+ hard_pending_compaction_bytes_limit = 1073741824ul;
+
+ BlockBasedTableOptions table_options;
+ table_options.block_cache =
+ (cache != nullptr) ? *cache : std::shared_ptr<Cache>();
+ table_options.cache_index_and_filter_blocks = true;
+ // Two level iterator to avoid LRU cache imbalance
+ table_options.index_type =
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+ table_factory.reset(new BlockBasedTableFactory(table_options));
+
+ return this;
+}
+
+#ifndef ROCKSDB_LITE
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(
+ uint64_t block_cache_size_mb) {
+ BlockBasedTableOptions block_based_options;
+ block_based_options.data_block_index_type =
+ BlockBasedTableOptions::kDataBlockBinaryAndHash;
+ block_based_options.data_block_hash_table_util_ratio = 0.75;
+ block_based_options.filter_policy.reset(NewBloomFilterPolicy(10));
+ block_based_options.block_cache =
+ NewLRUCache(static_cast<size_t>(block_cache_size_mb * 1024 * 1024));
+ table_factory.reset(new BlockBasedTableFactory(block_based_options));
+ memtable_prefix_bloom_size_ratio = 0.02;
+ memtable_whole_key_filtering = true;
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction(
+ uint64_t memtable_memory_budget) {
+ write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
+ // merge two memtables when flushing to L0
+ min_write_buffer_number_to_merge = 2;
+ // this means we'll use 50% extra memory in the worst case, but will reduce
+ // write stalls.
+ max_write_buffer_number = 6;
+ // start flushing L0->L1 as soon as possible. each file on level0 is
+ // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
+ // memtable_memory_budget.
+ level0_file_num_compaction_trigger = 2;
+ // doesn't really matter much, but we don't want to create too many files
+ target_file_size_base = memtable_memory_budget / 8;
+ // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
+ max_bytes_for_level_base = memtable_memory_budget;
+
+ // level style compaction
+ compaction_style = kCompactionStyleLevel;
+
+ // only compress levels >= 2
+ compression_per_level.resize(num_levels);
+ for (int i = 0; i < num_levels; ++i) {
+ if (i < 2) {
+ compression_per_level[i] = kNoCompression;
+ } else {
+ compression_per_level[i] =
+ LZ4_Supported()
+ ? kLZ4Compression
+ : (Snappy_Supported() ? kSnappyCompression : kNoCompression);
+ }
+ }
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction(
+ uint64_t memtable_memory_budget) {
+ write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
+ // merge two memtables when flushing to L0
+ min_write_buffer_number_to_merge = 2;
+ // this means we'll use 50% extra memory in the worst case, but will reduce
+ // write stalls.
+ max_write_buffer_number = 6;
+ // universal style compaction
+ compaction_style = kCompactionStyleUniversal;
+ compaction_options_universal.compression_size_percent = 80;
+ return this;
+}
+
+DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
+ max_background_jobs = total_threads;
+ env->SetBackgroundThreads(total_threads, Env::LOW);
+ env->SetBackgroundThreads(1, Env::HIGH);
+ return this;
+}
+
+#endif // !ROCKSDB_LITE
+
+ReadOptions::ReadOptions()
+ : snapshot(nullptr),
+ iterate_lower_bound(nullptr),
+ iterate_upper_bound(nullptr),
+ readahead_size(0),
+ max_skippable_internal_keys(0),
+ read_tier(kReadAllTier),
+ verify_checksums(true),
+ fill_cache(true),
+ tailing(false),
+ managed(false),
+ total_order_seek(false),
+ auto_prefix_mode(false),
+ prefix_same_as_start(false),
+ pin_data(false),
+ background_purge_on_iterator_cleanup(false),
+ ignore_range_deletions(false),
+ iter_start_seqnum(0),
+ timestamp(nullptr) {}
+
+ReadOptions::ReadOptions(bool cksum, bool cache)
+ : snapshot(nullptr),
+ iterate_lower_bound(nullptr),
+ iterate_upper_bound(nullptr),
+ readahead_size(0),
+ max_skippable_internal_keys(0),
+ read_tier(kReadAllTier),
+ verify_checksums(cksum),
+ fill_cache(cache),
+ tailing(false),
+ managed(false),
+ total_order_seek(false),
+ auto_prefix_mode(false),
+ prefix_same_as_start(false),
+ pin_data(false),
+ background_purge_on_iterator_cleanup(false),
+ ignore_range_deletions(false),
+ iter_start_seqnum(0),
+ timestamp(nullptr) {}
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_helper.cc b/src/rocksdb/options/options_helper.cc
new file mode 100644
index 000000000..bc36dbcfe
--- /dev/null
+++ b/src/rocksdb/options/options_helper.cc
@@ -0,0 +1,2124 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#include "options/options_helper.h"
+
+#include <cassert>
+#include <cctype>
+#include <cstdlib>
+#include <unordered_set>
+#include <vector>
+
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/rate_limiter.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/table.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "table/block_based/block_based_table_factory.h"
+#include "table/plain/plain_table_factory.h"
+#include "util/cast_util.h"
+#include "util/string_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
+ const MutableDBOptions& mutable_db_options) {
+ DBOptions options;
+
+ options.create_if_missing = immutable_db_options.create_if_missing;
+ options.create_missing_column_families =
+ immutable_db_options.create_missing_column_families;
+ options.error_if_exists = immutable_db_options.error_if_exists;
+ options.paranoid_checks = immutable_db_options.paranoid_checks;
+ options.env = immutable_db_options.env;
+ options.file_system = immutable_db_options.fs;
+ options.rate_limiter = immutable_db_options.rate_limiter;
+ options.sst_file_manager = immutable_db_options.sst_file_manager;
+ options.info_log = immutable_db_options.info_log;
+ options.info_log_level = immutable_db_options.info_log_level;
+ options.max_open_files = mutable_db_options.max_open_files;
+ options.max_file_opening_threads =
+ immutable_db_options.max_file_opening_threads;
+ options.max_total_wal_size = mutable_db_options.max_total_wal_size;
+ options.statistics = immutable_db_options.statistics;
+ options.use_fsync = immutable_db_options.use_fsync;
+ options.db_paths = immutable_db_options.db_paths;
+ options.db_log_dir = immutable_db_options.db_log_dir;
+ options.wal_dir = immutable_db_options.wal_dir;
+ options.delete_obsolete_files_period_micros =
+ mutable_db_options.delete_obsolete_files_period_micros;
+ options.max_background_jobs = mutable_db_options.max_background_jobs;
+ options.base_background_compactions =
+ mutable_db_options.base_background_compactions;
+ options.max_background_compactions =
+ mutable_db_options.max_background_compactions;
+ options.bytes_per_sync = mutable_db_options.bytes_per_sync;
+ options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync;
+ options.strict_bytes_per_sync = mutable_db_options.strict_bytes_per_sync;
+ options.max_subcompactions = immutable_db_options.max_subcompactions;
+ options.max_background_flushes = immutable_db_options.max_background_flushes;
+ options.max_log_file_size = immutable_db_options.max_log_file_size;
+ options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll;
+ options.keep_log_file_num = immutable_db_options.keep_log_file_num;
+ options.recycle_log_file_num = immutable_db_options.recycle_log_file_num;
+ options.max_manifest_file_size = immutable_db_options.max_manifest_file_size;
+ options.table_cache_numshardbits =
+ immutable_db_options.table_cache_numshardbits;
+ options.WAL_ttl_seconds = immutable_db_options.wal_ttl_seconds;
+ options.WAL_size_limit_MB = immutable_db_options.wal_size_limit_mb;
+ options.manifest_preallocation_size =
+ immutable_db_options.manifest_preallocation_size;
+ options.allow_mmap_reads = immutable_db_options.allow_mmap_reads;
+ options.allow_mmap_writes = immutable_db_options.allow_mmap_writes;
+ options.use_direct_reads = immutable_db_options.use_direct_reads;
+ options.use_direct_io_for_flush_and_compaction =
+ immutable_db_options.use_direct_io_for_flush_and_compaction;
+ options.allow_fallocate = immutable_db_options.allow_fallocate;
+ options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec;
+ options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec;
+ options.stats_persist_period_sec =
+ mutable_db_options.stats_persist_period_sec;
+ options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk;
+ options.stats_history_buffer_size =
+ mutable_db_options.stats_history_buffer_size;
+ options.advise_random_on_open = immutable_db_options.advise_random_on_open;
+ options.db_write_buffer_size = immutable_db_options.db_write_buffer_size;
+ options.write_buffer_manager = immutable_db_options.write_buffer_manager;
+ options.access_hint_on_compaction_start =
+ immutable_db_options.access_hint_on_compaction_start;
+ options.new_table_reader_for_compaction_inputs =
+ immutable_db_options.new_table_reader_for_compaction_inputs;
+ options.compaction_readahead_size =
+ mutable_db_options.compaction_readahead_size;
+ options.random_access_max_buffer_size =
+ immutable_db_options.random_access_max_buffer_size;
+ options.writable_file_max_buffer_size =
+ mutable_db_options.writable_file_max_buffer_size;
+ options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex;
+ options.listeners = immutable_db_options.listeners;
+ options.enable_thread_tracking = immutable_db_options.enable_thread_tracking;
+ options.delayed_write_rate = mutable_db_options.delayed_write_rate;
+ options.enable_pipelined_write = immutable_db_options.enable_pipelined_write;
+ options.unordered_write = immutable_db_options.unordered_write;
+ options.allow_concurrent_memtable_write =
+ immutable_db_options.allow_concurrent_memtable_write;
+ options.enable_write_thread_adaptive_yield =
+ immutable_db_options.enable_write_thread_adaptive_yield;
+ options.max_write_batch_group_size_bytes =
+ immutable_db_options.max_write_batch_group_size_bytes;
+ options.write_thread_max_yield_usec =
+ immutable_db_options.write_thread_max_yield_usec;
+ options.write_thread_slow_yield_usec =
+ immutable_db_options.write_thread_slow_yield_usec;
+ options.skip_stats_update_on_db_open =
+ immutable_db_options.skip_stats_update_on_db_open;
+ options.skip_checking_sst_file_sizes_on_db_open =
+ immutable_db_options.skip_checking_sst_file_sizes_on_db_open;
+ options.wal_recovery_mode = immutable_db_options.wal_recovery_mode;
+ options.allow_2pc = immutable_db_options.allow_2pc;
+ options.row_cache = immutable_db_options.row_cache;
+#ifndef ROCKSDB_LITE
+ options.wal_filter = immutable_db_options.wal_filter;
+#endif // ROCKSDB_LITE
+ options.fail_if_options_file_error =
+ immutable_db_options.fail_if_options_file_error;
+ options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
+ options.avoid_flush_during_recovery =
+ immutable_db_options.avoid_flush_during_recovery;
+ options.avoid_flush_during_shutdown =
+ mutable_db_options.avoid_flush_during_shutdown;
+ options.allow_ingest_behind =
+ immutable_db_options.allow_ingest_behind;
+ options.preserve_deletes =
+ immutable_db_options.preserve_deletes;
+ options.two_write_queues = immutable_db_options.two_write_queues;
+ options.manual_wal_flush = immutable_db_options.manual_wal_flush;
+ options.atomic_flush = immutable_db_options.atomic_flush;
+ options.avoid_unnecessary_blocking_io =
+ immutable_db_options.avoid_unnecessary_blocking_io;
+ options.log_readahead_size = immutable_db_options.log_readahead_size;
+ options.sst_file_checksum_func = immutable_db_options.sst_file_checksum_func;
+ return options;
+}
+
+ColumnFamilyOptions BuildColumnFamilyOptions(
+ const ColumnFamilyOptions& options,
+ const MutableCFOptions& mutable_cf_options) {
+ ColumnFamilyOptions cf_opts(options);
+
+ // Memtable related options
+ cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
+ cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
+ cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
+ cf_opts.memtable_prefix_bloom_size_ratio =
+ mutable_cf_options.memtable_prefix_bloom_size_ratio;
+ cf_opts.memtable_whole_key_filtering =
+ mutable_cf_options.memtable_whole_key_filtering;
+ cf_opts.memtable_huge_page_size = mutable_cf_options.memtable_huge_page_size;
+ cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;
+ cf_opts.inplace_update_num_locks =
+ mutable_cf_options.inplace_update_num_locks;
+ cf_opts.prefix_extractor = mutable_cf_options.prefix_extractor;
+
+ // Compaction related options
+ cf_opts.disable_auto_compactions =
+ mutable_cf_options.disable_auto_compactions;
+ cf_opts.soft_pending_compaction_bytes_limit =
+ mutable_cf_options.soft_pending_compaction_bytes_limit;
+ cf_opts.hard_pending_compaction_bytes_limit =
+ mutable_cf_options.hard_pending_compaction_bytes_limit;
+ cf_opts.level0_file_num_compaction_trigger =
+ mutable_cf_options.level0_file_num_compaction_trigger;
+ cf_opts.level0_slowdown_writes_trigger =
+ mutable_cf_options.level0_slowdown_writes_trigger;
+ cf_opts.level0_stop_writes_trigger =
+ mutable_cf_options.level0_stop_writes_trigger;
+ cf_opts.max_compaction_bytes = mutable_cf_options.max_compaction_bytes;
+ cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base;
+ cf_opts.target_file_size_multiplier =
+ mutable_cf_options.target_file_size_multiplier;
+ cf_opts.max_bytes_for_level_base =
+ mutable_cf_options.max_bytes_for_level_base;
+ cf_opts.max_bytes_for_level_multiplier =
+ mutable_cf_options.max_bytes_for_level_multiplier;
+ cf_opts.ttl = mutable_cf_options.ttl;
+ cf_opts.periodic_compaction_seconds =
+ mutable_cf_options.periodic_compaction_seconds;
+
+ cf_opts.max_bytes_for_level_multiplier_additional.clear();
+ for (auto value :
+ mutable_cf_options.max_bytes_for_level_multiplier_additional) {
+ cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
+ }
+
+ cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo;
+ cf_opts.compaction_options_universal =
+ mutable_cf_options.compaction_options_universal;
+
+ // Misc options
+ cf_opts.max_sequential_skip_in_iterations =
+ mutable_cf_options.max_sequential_skip_in_iterations;
+ cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
+ cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
+ cf_opts.compression = mutable_cf_options.compression;
+ cf_opts.sample_for_compression = mutable_cf_options.sample_for_compression;
+
+ cf_opts.table_factory = options.table_factory;
+ // TODO(yhchiang): find some way to handle the following derived options
+ // * max_file_size
+
+ return cf_opts;
+}
+
+std::map<CompactionStyle, std::string>
+ OptionsHelper::compaction_style_to_string = {
+ {kCompactionStyleLevel, "kCompactionStyleLevel"},
+ {kCompactionStyleUniversal, "kCompactionStyleUniversal"},
+ {kCompactionStyleFIFO, "kCompactionStyleFIFO"},
+ {kCompactionStyleNone, "kCompactionStyleNone"}};
+
+std::map<CompactionPri, std::string> OptionsHelper::compaction_pri_to_string = {
+ {kByCompensatedSize, "kByCompensatedSize"},
+ {kOldestLargestSeqFirst, "kOldestLargestSeqFirst"},
+ {kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"},
+ {kMinOverlappingRatio, "kMinOverlappingRatio"}};
+
+std::map<CompactionStopStyle, std::string>
+ OptionsHelper::compaction_stop_style_to_string = {
+ {kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"},
+ {kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}};
+
+std::unordered_map<std::string, ChecksumType>
+ OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum},
+ {"kCRC32c", kCRC32c},
+ {"kxxHash", kxxHash},
+ {"kxxHash64", kxxHash64}};
+
+std::unordered_map<std::string, CompressionType>
+ OptionsHelper::compression_type_string_map = {
+ {"kNoCompression", kNoCompression},
+ {"kSnappyCompression", kSnappyCompression},
+ {"kZlibCompression", kZlibCompression},
+ {"kBZip2Compression", kBZip2Compression},
+ {"kLZ4Compression", kLZ4Compression},
+ {"kLZ4HCCompression", kLZ4HCCompression},
+ {"kXpressCompression", kXpressCompression},
+ {"kZSTD", kZSTD},
+ {"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
+ {"kDisableCompressionOption", kDisableCompressionOption}};
+#ifndef ROCKSDB_LITE
+
+const std::string kNameComparator = "comparator";
+const std::string kNameEnv = "env";
+const std::string kNameMergeOperator = "merge_operator";
+
+template <typename T>
+Status GetStringFromStruct(
+ std::string* opt_string, const T& options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_info,
+ const std::string& delimiter);
+
+namespace {
+template <typename T>
+bool ParseEnum(const std::unordered_map<std::string, T>& type_map,
+ const std::string& type, T* value) {
+ auto iter = type_map.find(type);
+ if (iter != type_map.end()) {
+ *value = iter->second;
+ return true;
+ }
+ return false;
+}
+
+template <typename T>
+bool SerializeEnum(const std::unordered_map<std::string, T>& type_map,
+ const T& type, std::string* value) {
+ for (const auto& pair : type_map) {
+ if (pair.second == type) {
+ *value = pair.first;
+ return true;
+ }
+ }
+ return false;
+}
+
+bool SerializeVectorCompressionType(const std::vector<CompressionType>& types,
+ std::string* value) {
+ std::stringstream ss;
+ bool result;
+ for (size_t i = 0; i < types.size(); ++i) {
+ if (i > 0) {
+ ss << ':';
+ }
+ std::string string_type;
+ result = SerializeEnum<CompressionType>(compression_type_string_map,
+ types[i], &string_type);
+ if (result == false) {
+ return result;
+ }
+ ss << string_type;
+ }
+ *value = ss.str();
+ return true;
+}
+
+bool ParseVectorCompressionType(
+ const std::string& value,
+ std::vector<CompressionType>* compression_per_level) {
+ compression_per_level->clear();
+ size_t start = 0;
+ while (start < value.size()) {
+ size_t end = value.find(':', start);
+ bool is_ok;
+ CompressionType type;
+ if (end == std::string::npos) {
+ is_ok = ParseEnum<CompressionType>(compression_type_string_map,
+ value.substr(start), &type);
+ if (!is_ok) {
+ return false;
+ }
+ compression_per_level->emplace_back(type);
+ break;
+ } else {
+ is_ok = ParseEnum<CompressionType>(
+ compression_type_string_map, value.substr(start, end - start), &type);
+ if (!is_ok) {
+ return false;
+ }
+ compression_per_level->emplace_back(type);
+ start = end + 1;
+ }
+ }
+ return true;
+}
+
+// This is to handle backward compatibility, where compaction_options_fifo
+// could be assigned a single scalar value, say, like "23", which would be
+// assigned to max_table_files_size.
+bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str,
+ CompactionOptionsFIFO* options) {
+ if (opt_str.find("=") != std::string::npos) {
+ // New format. Go do your new parsing using ParseStructOptions.
+ return false;
+ }
+
+ // Old format. Parse just a single uint64_t value.
+ options->max_table_files_size = ParseUint64(opt_str);
+ return true;
+}
+
+template <typename T>
+bool SerializeStruct(
+ const T& options, std::string* value,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
+ std::string opt_str;
+ Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";");
+ if (!s.ok()) {
+ return false;
+ }
+ *value = "{" + opt_str + "}";
+ return true;
+}
+
+template <typename T>
+bool ParseSingleStructOption(
+ const std::string& opt_val_str, T* options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
+ size_t end = opt_val_str.find('=');
+ std::string key = opt_val_str.substr(0, end);
+ std::string value = opt_val_str.substr(end + 1);
+ auto iter = type_info_map.find(key);
+ if (iter == type_info_map.end()) {
+ return false;
+ }
+ const auto& opt_info = iter->second;
+ if (opt_info.verification == OptionVerificationType::kDeprecated) {
+ // Should also skip deprecated sub-options such as
+ // fifo_compaction_options_type_info.ttl
+ return true;
+ }
+ return ParseOptionHelper(
+ reinterpret_cast<char*>(options) + opt_info.mutable_offset, opt_info.type,
+ value);
+}
+
+template <typename T>
+bool ParseStructOptions(
+ const std::string& opt_str, T* options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_info_map) {
+ assert(!opt_str.empty());
+
+ size_t start = 0;
+ if (opt_str[0] == '{') {
+ start++;
+ }
+ while ((start != std::string::npos) && (start < opt_str.size())) {
+ if (opt_str[start] == '}') {
+ break;
+ }
+ size_t end = opt_str.find(';', start);
+ size_t len = (end == std::string::npos) ? end : end - start;
+ if (!ParseSingleStructOption(opt_str.substr(start, len), options,
+ type_info_map)) {
+ return false;
+ }
+ start = (end == std::string::npos) ? end : end + 1;
+ }
+ return true;
+}
+} // anonymouse namespace
+
+bool ParseSliceTransformHelper(
+ const std::string& kFixedPrefixName, const std::string& kCappedPrefixName,
+ const std::string& value,
+ std::shared_ptr<const SliceTransform>* slice_transform) {
+ const char* no_op_name = "rocksdb.Noop";
+ size_t no_op_length = strlen(no_op_name);
+ auto& pe_value = value;
+ if (pe_value.size() > kFixedPrefixName.size() &&
+ pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
+ int prefix_length = ParseInt(trim(value.substr(kFixedPrefixName.size())));
+ slice_transform->reset(NewFixedPrefixTransform(prefix_length));
+ } else if (pe_value.size() > kCappedPrefixName.size() &&
+ pe_value.compare(0, kCappedPrefixName.size(), kCappedPrefixName) ==
+ 0) {
+ int prefix_length =
+ ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
+ slice_transform->reset(NewCappedPrefixTransform(prefix_length));
+ } else if (pe_value.size() == no_op_length &&
+ pe_value.compare(0, no_op_length, no_op_name) == 0) {
+ const SliceTransform* no_op_transform = NewNoopTransform();
+ slice_transform->reset(no_op_transform);
+ } else if (value == kNullptrString) {
+ slice_transform->reset();
+ } else {
+ return false;
+ }
+
+ return true;
+}
+
+bool ParseSliceTransform(
+ const std::string& value,
+ std::shared_ptr<const SliceTransform>* slice_transform) {
+ // While we normally don't convert the string representation of a
+ // pointer-typed option into its instance, here we do so for backward
+ // compatibility as we allow this action in SetOption().
+
+ // TODO(yhchiang): A possible better place for these serialization /
+ // deserialization is inside the class definition of pointer-typed
+ // option itself, but this requires a bigger change of public API.
+ bool result =
+ ParseSliceTransformHelper("fixed:", "capped:", value, slice_transform);
+ if (result) {
+ return result;
+ }
+ result = ParseSliceTransformHelper(
+ "rocksdb.FixedPrefix.", "rocksdb.CappedPrefix.", value, slice_transform);
+ if (result) {
+ return result;
+ }
+ // TODO(yhchiang): we can further support other default
+ // SliceTransforms here.
+ return false;
+}
+
+bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
+ const std::string& value) {
+ switch (opt_type) {
+ case OptionType::kBoolean:
+ *reinterpret_cast<bool*>(opt_address) = ParseBoolean("", value);
+ break;
+ case OptionType::kInt:
+ *reinterpret_cast<int*>(opt_address) = ParseInt(value);
+ break;
+ case OptionType::kInt32T:
+ *reinterpret_cast<int32_t*>(opt_address) = ParseInt32(value);
+ break;
+ case OptionType::kInt64T:
+ PutUnaligned(reinterpret_cast<int64_t*>(opt_address), ParseInt64(value));
+ break;
+ case OptionType::kVectorInt:
+ *reinterpret_cast<std::vector<int>*>(opt_address) = ParseVectorInt(value);
+ break;
+ case OptionType::kUInt:
+ *reinterpret_cast<unsigned int*>(opt_address) = ParseUint32(value);
+ break;
+ case OptionType::kUInt32T:
+ *reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value);
+ break;
+ case OptionType::kUInt64T:
+ PutUnaligned(reinterpret_cast<uint64_t*>(opt_address), ParseUint64(value));
+ break;
+ case OptionType::kSizeT:
+ PutUnaligned(reinterpret_cast<size_t*>(opt_address), ParseSizeT(value));
+ break;
+ case OptionType::kString:
+ *reinterpret_cast<std::string*>(opt_address) = value;
+ break;
+ case OptionType::kDouble:
+ *reinterpret_cast<double*>(opt_address) = ParseDouble(value);
+ break;
+ case OptionType::kCompactionStyle:
+ return ParseEnum<CompactionStyle>(
+ compaction_style_string_map, value,
+ reinterpret_cast<CompactionStyle*>(opt_address));
+ case OptionType::kCompactionPri:
+ return ParseEnum<CompactionPri>(
+ compaction_pri_string_map, value,
+ reinterpret_cast<CompactionPri*>(opt_address));
+ case OptionType::kCompressionType:
+ return ParseEnum<CompressionType>(
+ compression_type_string_map, value,
+ reinterpret_cast<CompressionType*>(opt_address));
+ case OptionType::kVectorCompressionType:
+ return ParseVectorCompressionType(
+ value, reinterpret_cast<std::vector<CompressionType>*>(opt_address));
+ case OptionType::kSliceTransform:
+ return ParseSliceTransform(
+ value, reinterpret_cast<std::shared_ptr<const SliceTransform>*>(
+ opt_address));
+ case OptionType::kChecksumType:
+ return ParseEnum<ChecksumType>(
+ checksum_type_string_map, value,
+ reinterpret_cast<ChecksumType*>(opt_address));
+ case OptionType::kBlockBasedTableIndexType:
+ return ParseEnum<BlockBasedTableOptions::IndexType>(
+ block_base_table_index_type_string_map, value,
+ reinterpret_cast<BlockBasedTableOptions::IndexType*>(opt_address));
+ case OptionType::kBlockBasedTableDataBlockIndexType:
+ return ParseEnum<BlockBasedTableOptions::DataBlockIndexType>(
+ block_base_table_data_block_index_type_string_map, value,
+ reinterpret_cast<BlockBasedTableOptions::DataBlockIndexType*>(
+ opt_address));
+ case OptionType::kBlockBasedTableIndexShorteningMode:
+ return ParseEnum<BlockBasedTableOptions::IndexShorteningMode>(
+ block_base_table_index_shortening_mode_string_map, value,
+ reinterpret_cast<BlockBasedTableOptions::IndexShorteningMode*>(
+ opt_address));
+ case OptionType::kEncodingType:
+ return ParseEnum<EncodingType>(
+ encoding_type_string_map, value,
+ reinterpret_cast<EncodingType*>(opt_address));
+ case OptionType::kWALRecoveryMode:
+ return ParseEnum<WALRecoveryMode>(
+ wal_recovery_mode_string_map, value,
+ reinterpret_cast<WALRecoveryMode*>(opt_address));
+ case OptionType::kAccessHint:
+ return ParseEnum<DBOptions::AccessHint>(
+ access_hint_string_map, value,
+ reinterpret_cast<DBOptions::AccessHint*>(opt_address));
+ case OptionType::kInfoLogLevel:
+ return ParseEnum<InfoLogLevel>(
+ info_log_level_string_map, value,
+ reinterpret_cast<InfoLogLevel*>(opt_address));
+ case OptionType::kCompactionOptionsFIFO: {
+ if (!FIFOCompactionOptionsSpecialCase(
+ value, reinterpret_cast<CompactionOptionsFIFO*>(opt_address))) {
+ return ParseStructOptions<CompactionOptionsFIFO>(
+ value, reinterpret_cast<CompactionOptionsFIFO*>(opt_address),
+ fifo_compaction_options_type_info);
+ }
+ return true;
+ }
+ case OptionType::kLRUCacheOptions: {
+ return ParseStructOptions<LRUCacheOptions>(value,
+ reinterpret_cast<LRUCacheOptions*>(opt_address),
+ lru_cache_options_type_info);
+ }
+ case OptionType::kCompactionOptionsUniversal:
+ return ParseStructOptions<CompactionOptionsUniversal>(
+ value, reinterpret_cast<CompactionOptionsUniversal*>(opt_address),
+ universal_compaction_options_type_info);
+ case OptionType::kCompactionStopStyle:
+ return ParseEnum<CompactionStopStyle>(
+ compaction_stop_style_string_map, value,
+ reinterpret_cast<CompactionStopStyle*>(opt_address));
+ default:
+ return false;
+ }
+ return true;
+}
+
+bool SerializeSingleOptionHelper(const char* opt_address,
+ const OptionType opt_type,
+ std::string* value) {
+
+ assert(value);
+ switch (opt_type) {
+ case OptionType::kBoolean:
+ *value = *(reinterpret_cast<const bool*>(opt_address)) ? "true" : "false";
+ break;
+ case OptionType::kInt:
+ *value = ToString(*(reinterpret_cast<const int*>(opt_address)));
+ break;
+ case OptionType::kInt32T:
+ *value = ToString(*(reinterpret_cast<const int32_t*>(opt_address)));
+ break;
+ case OptionType::kInt64T:
+ {
+ int64_t v;
+ GetUnaligned(reinterpret_cast<const int64_t*>(opt_address), &v);
+ *value = ToString(v);
+ }
+ break;
+ case OptionType::kVectorInt:
+ return SerializeIntVector(
+ *reinterpret_cast<const std::vector<int>*>(opt_address), value);
+ case OptionType::kUInt:
+ *value = ToString(*(reinterpret_cast<const unsigned int*>(opt_address)));
+ break;
+ case OptionType::kUInt32T:
+ *value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address)));
+ break;
+ case OptionType::kUInt64T:
+ {
+ uint64_t v;
+ GetUnaligned(reinterpret_cast<const uint64_t*>(opt_address), &v);
+ *value = ToString(v);
+ }
+ break;
+ case OptionType::kSizeT:
+ {
+ size_t v;
+ GetUnaligned(reinterpret_cast<const size_t*>(opt_address), &v);
+ *value = ToString(v);
+ }
+ break;
+ case OptionType::kDouble:
+ *value = ToString(*(reinterpret_cast<const double*>(opt_address)));
+ break;
+ case OptionType::kString:
+ *value = EscapeOptionString(
+ *(reinterpret_cast<const std::string*>(opt_address)));
+ break;
+ case OptionType::kCompactionStyle:
+ return SerializeEnum<CompactionStyle>(
+ compaction_style_string_map,
+ *(reinterpret_cast<const CompactionStyle*>(opt_address)), value);
+ case OptionType::kCompactionPri:
+ return SerializeEnum<CompactionPri>(
+ compaction_pri_string_map,
+ *(reinterpret_cast<const CompactionPri*>(opt_address)), value);
+ case OptionType::kCompressionType:
+ return SerializeEnum<CompressionType>(
+ compression_type_string_map,
+ *(reinterpret_cast<const CompressionType*>(opt_address)), value);
+ case OptionType::kVectorCompressionType:
+ return SerializeVectorCompressionType(
+ *(reinterpret_cast<const std::vector<CompressionType>*>(opt_address)),
+ value);
+ break;
+ case OptionType::kSliceTransform: {
+ const auto* slice_transform_ptr =
+ reinterpret_cast<const std::shared_ptr<const SliceTransform>*>(
+ opt_address);
+ *value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name()
+ : kNullptrString;
+ break;
+ }
+ case OptionType::kTableFactory: {
+ const auto* table_factory_ptr =
+ reinterpret_cast<const std::shared_ptr<const TableFactory>*>(
+ opt_address);
+ *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name()
+ : kNullptrString;
+ break;
+ }
+ case OptionType::kComparator: {
+ // it's a const pointer of const Comparator*
+ const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address);
+ // Since the user-specified comparator will be wrapped by
+ // InternalKeyComparator, we should persist the user-specified one
+ // instead of InternalKeyComparator.
+ if (*ptr == nullptr) {
+ *value = kNullptrString;
+ } else {
+ const Comparator* root_comp = (*ptr)->GetRootComparator();
+ if (root_comp == nullptr) {
+ root_comp = (*ptr);
+ }
+ *value = root_comp->Name();
+ }
+ break;
+ }
+ case OptionType::kCompactionFilter: {
+ // it's a const pointer of const CompactionFilter*
+ const auto* ptr =
+ reinterpret_cast<const CompactionFilter* const*>(opt_address);
+ *value = *ptr ? (*ptr)->Name() : kNullptrString;
+ break;
+ }
+ case OptionType::kCompactionFilterFactory: {
+ const auto* ptr =
+ reinterpret_cast<const std::shared_ptr<CompactionFilterFactory>*>(
+ opt_address);
+ *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
+ break;
+ }
+ case OptionType::kMemTableRepFactory: {
+ const auto* ptr =
+ reinterpret_cast<const std::shared_ptr<MemTableRepFactory>*>(
+ opt_address);
+ *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
+ break;
+ }
+ case OptionType::kMergeOperator: {
+ const auto* ptr =
+ reinterpret_cast<const std::shared_ptr<MergeOperator>*>(opt_address);
+ *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
+ break;
+ }
+ case OptionType::kFilterPolicy: {
+ const auto* ptr =
+ reinterpret_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
+ *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
+ break;
+ }
+ case OptionType::kChecksumType:
+ return SerializeEnum<ChecksumType>(
+ checksum_type_string_map,
+ *reinterpret_cast<const ChecksumType*>(opt_address), value);
+ case OptionType::kBlockBasedTableIndexType:
+ return SerializeEnum<BlockBasedTableOptions::IndexType>(
+ block_base_table_index_type_string_map,
+ *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
+ opt_address),
+ value);
+ case OptionType::kBlockBasedTableDataBlockIndexType:
+ return SerializeEnum<BlockBasedTableOptions::DataBlockIndexType>(
+ block_base_table_data_block_index_type_string_map,
+ *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
+ opt_address),
+ value);
+ case OptionType::kBlockBasedTableIndexShorteningMode:
+ return SerializeEnum<BlockBasedTableOptions::IndexShorteningMode>(
+ block_base_table_index_shortening_mode_string_map,
+ *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
+ opt_address),
+ value);
+ case OptionType::kFlushBlockPolicyFactory: {
+ const auto* ptr =
+ reinterpret_cast<const std::shared_ptr<FlushBlockPolicyFactory>*>(
+ opt_address);
+ *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
+ break;
+ }
+ case OptionType::kEncodingType:
+ return SerializeEnum<EncodingType>(
+ encoding_type_string_map,
+ *reinterpret_cast<const EncodingType*>(opt_address), value);
+ case OptionType::kWALRecoveryMode:
+ return SerializeEnum<WALRecoveryMode>(
+ wal_recovery_mode_string_map,
+ *reinterpret_cast<const WALRecoveryMode*>(opt_address), value);
+ case OptionType::kAccessHint:
+ return SerializeEnum<DBOptions::AccessHint>(
+ access_hint_string_map,
+ *reinterpret_cast<const DBOptions::AccessHint*>(opt_address), value);
+ case OptionType::kInfoLogLevel:
+ return SerializeEnum<InfoLogLevel>(
+ info_log_level_string_map,
+ *reinterpret_cast<const InfoLogLevel*>(opt_address), value);
+ case OptionType::kCompactionOptionsFIFO:
+ return SerializeStruct<CompactionOptionsFIFO>(
+ *reinterpret_cast<const CompactionOptionsFIFO*>(opt_address), value,
+ fifo_compaction_options_type_info);
+ case OptionType::kCompactionOptionsUniversal:
+ return SerializeStruct<CompactionOptionsUniversal>(
+ *reinterpret_cast<const CompactionOptionsUniversal*>(opt_address),
+ value, universal_compaction_options_type_info);
+ case OptionType::kCompactionStopStyle:
+ return SerializeEnum<CompactionStopStyle>(
+ compaction_stop_style_string_map,
+ *reinterpret_cast<const CompactionStopStyle*>(opt_address), value);
+ default:
+ return false;
+ }
+ return true;
+}
+
+Status GetMutableOptionsFromStrings(
+ const MutableCFOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ Logger* info_log, MutableCFOptions* new_options) {
+ assert(new_options);
+ *new_options = base_options;
+ for (const auto& o : options_map) {
+ try {
+ auto iter = cf_options_type_info.find(o.first);
+ if (iter == cf_options_type_info.end()) {
+ return Status::InvalidArgument("Unrecognized option: " + o.first);
+ }
+ const auto& opt_info = iter->second;
+ if (!opt_info.is_mutable) {
+ return Status::InvalidArgument("Option not changeable: " + o.first);
+ }
+ if (opt_info.verification == OptionVerificationType::kDeprecated) {
+ // log warning when user tries to set a deprecated option but don't fail
+ // the call for compatibility.
+ ROCKS_LOG_WARN(info_log, "%s is a deprecated option and cannot be set",
+ o.first.c_str());
+ continue;
+ }
+ bool is_ok = ParseOptionHelper(
+ reinterpret_cast<char*>(new_options) + opt_info.mutable_offset,
+ opt_info.type, o.second);
+ if (!is_ok) {
+ return Status::InvalidArgument("Error parsing " + o.first);
+ }
+ } catch (std::exception& e) {
+ return Status::InvalidArgument("Error parsing " + o.first + ":" +
+ std::string(e.what()));
+ }
+ }
+ return Status::OK();
+}
+
+Status GetMutableDBOptionsFromStrings(
+ const MutableDBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ MutableDBOptions* new_options) {
+ assert(new_options);
+ *new_options = base_options;
+ for (const auto& o : options_map) {
+ try {
+ auto iter = db_options_type_info.find(o.first);
+ if (iter == db_options_type_info.end()) {
+ return Status::InvalidArgument("Unrecognized option: " + o.first);
+ }
+ const auto& opt_info = iter->second;
+ if (!opt_info.is_mutable) {
+ return Status::InvalidArgument("Option not changeable: " + o.first);
+ }
+ bool is_ok = ParseOptionHelper(
+ reinterpret_cast<char*>(new_options) + opt_info.mutable_offset,
+ opt_info.type, o.second);
+ if (!is_ok) {
+ return Status::InvalidArgument("Error parsing " + o.first);
+ }
+ } catch (std::exception& e) {
+ return Status::InvalidArgument("Error parsing " + o.first + ":" +
+ std::string(e.what()));
+ }
+ }
+ return Status::OK();
+}
+
+Status StringToMap(const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map) {
+ assert(opts_map);
+ // Example:
+ // opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
+ // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
+ size_t pos = 0;
+ std::string opts = trim(opts_str);
+ while (pos < opts.size()) {
+ size_t eq_pos = opts.find('=', pos);
+ if (eq_pos == std::string::npos) {
+ return Status::InvalidArgument("Mismatched key value pair, '=' expected");
+ }
+ std::string key = trim(opts.substr(pos, eq_pos - pos));
+ if (key.empty()) {
+ return Status::InvalidArgument("Empty key found");
+ }
+
+ // skip space after '=' and look for '{' for possible nested options
+ pos = eq_pos + 1;
+ while (pos < opts.size() && isspace(opts[pos])) {
+ ++pos;
+ }
+ // Empty value at the end
+ if (pos >= opts.size()) {
+ (*opts_map)[key] = "";
+ break;
+ }
+ if (opts[pos] == '{') {
+ int count = 1;
+ size_t brace_pos = pos + 1;
+ while (brace_pos < opts.size()) {
+ if (opts[brace_pos] == '{') {
+ ++count;
+ } else if (opts[brace_pos] == '}') {
+ --count;
+ if (count == 0) {
+ break;
+ }
+ }
+ ++brace_pos;
+ }
+ // found the matching closing brace
+ if (count == 0) {
+ (*opts_map)[key] = trim(opts.substr(pos + 1, brace_pos - pos - 1));
+ // skip all whitespace and move to the next ';'
+ // brace_pos points to the next position after the matching '}'
+ pos = brace_pos + 1;
+ while (pos < opts.size() && isspace(opts[pos])) {
+ ++pos;
+ }
+ if (pos < opts.size() && opts[pos] != ';') {
+ return Status::InvalidArgument(
+ "Unexpected chars after nested options");
+ }
+ ++pos;
+ } else {
+ return Status::InvalidArgument(
+ "Mismatched curly braces for nested options");
+ }
+ } else {
+ size_t sc_pos = opts.find(';', pos);
+ if (sc_pos == std::string::npos) {
+ (*opts_map)[key] = trim(opts.substr(pos));
+ // It either ends with a trailing semi-colon or the last key-value pair
+ break;
+ } else {
+ (*opts_map)[key] = trim(opts.substr(pos, sc_pos - pos));
+ }
+ pos = sc_pos + 1;
+ }
+ }
+
+ return Status::OK();
+}
+
+Status ParseCompressionOptions(const std::string& value, const std::string& name,
+ CompressionOptions& compression_opts) {
+ size_t start = 0;
+ size_t end = value.find(':');
+ if (end == std::string::npos) {
+ return Status::InvalidArgument("unable to parse the specified CF option " +
+ name);
+ }
+ compression_opts.window_bits = ParseInt(value.substr(start, end - start));
+ start = end + 1;
+ end = value.find(':', start);
+ if (end == std::string::npos) {
+ return Status::InvalidArgument("unable to parse the specified CF option " +
+ name);
+ }
+ compression_opts.level = ParseInt(value.substr(start, end - start));
+ start = end + 1;
+ if (start >= value.size()) {
+ return Status::InvalidArgument("unable to parse the specified CF option " +
+ name);
+ }
+ end = value.find(':', start);
+ compression_opts.strategy =
+ ParseInt(value.substr(start, value.size() - start));
+ // max_dict_bytes is optional for backwards compatibility
+ if (end != std::string::npos) {
+ start = end + 1;
+ if (start >= value.size()) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.max_dict_bytes =
+ ParseInt(value.substr(start, value.size() - start));
+ end = value.find(':', start);
+ }
+ // zstd_max_train_bytes is optional for backwards compatibility
+ if (end != std::string::npos) {
+ start = end + 1;
+ if (start >= value.size()) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.zstd_max_train_bytes =
+ ParseInt(value.substr(start, value.size() - start));
+ end = value.find(':', start);
+ }
+ // enabled is optional for backwards compatibility
+ if (end != std::string::npos) {
+ start = end + 1;
+ if (start >= value.size()) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.enabled =
+ ParseBoolean("", value.substr(start, value.size() - start));
+ }
+ return Status::OK();
+}
+
+Status ParseColumnFamilyOption(const std::string& name,
+ const std::string& org_value,
+ ColumnFamilyOptions* new_options,
+ bool input_strings_escaped = false) {
+ const std::string& value =
+ input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
+ try {
+ if (name == "block_based_table_factory") {
+ // Nested options
+ BlockBasedTableOptions table_opt, base_table_options;
+ BlockBasedTableFactory* block_based_table_factory =
+ static_cast_with_check<BlockBasedTableFactory, TableFactory>(
+ new_options->table_factory.get());
+ if (block_based_table_factory != nullptr) {
+ base_table_options = block_based_table_factory->table_options();
+ }
+ Status table_opt_s = GetBlockBasedTableOptionsFromString(
+ base_table_options, value, &table_opt);
+ if (!table_opt_s.ok()) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
+ } else if (name == "plain_table_factory") {
+ // Nested options
+ PlainTableOptions table_opt, base_table_options;
+ PlainTableFactory* plain_table_factory =
+ static_cast_with_check<PlainTableFactory, TableFactory>(
+ new_options->table_factory.get());
+ if (plain_table_factory != nullptr) {
+ base_table_options = plain_table_factory->table_options();
+ }
+ Status table_opt_s = GetPlainTableOptionsFromString(
+ base_table_options, value, &table_opt);
+ if (!table_opt_s.ok()) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ new_options->table_factory.reset(NewPlainTableFactory(table_opt));
+ } else if (name == "memtable") {
+ std::unique_ptr<MemTableRepFactory> new_mem_factory;
+ Status mem_factory_s =
+ GetMemTableRepFactoryFromString(value, &new_mem_factory);
+ if (!mem_factory_s.ok()) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ new_options->memtable_factory.reset(new_mem_factory.release());
+ } else if (name == "bottommost_compression_opts") {
+ Status s = ParseCompressionOptions(
+ value, name, new_options->bottommost_compression_opts);
+ if (!s.ok()) {
+ return s;
+ }
+ } else if (name == "compression_opts") {
+ Status s =
+ ParseCompressionOptions(value, name, new_options->compression_opts);
+ if (!s.ok()) {
+ return s;
+ }
+ } else {
+ if (name == kNameComparator) {
+ // Try to get comparator from object registry first.
+ // Only support static comparator for now.
+ Status status = ObjectRegistry::NewInstance()->NewStaticObject(
+ value, &new_options->comparator);
+ if (status.ok()) {
+ return status;
+ }
+ } else if (name == kNameMergeOperator) {
+ // Try to get merge operator from object registry first.
+ std::shared_ptr<MergeOperator> mo;
+ Status status =
+ ObjectRegistry::NewInstance()->NewSharedObject<MergeOperator>(
+ value, &new_options->merge_operator);
+ // Only support static comparator for now.
+ if (status.ok()) {
+ return status;
+ }
+ }
+
+ auto iter = cf_options_type_info.find(name);
+ if (iter == cf_options_type_info.end()) {
+ return Status::InvalidArgument(
+ "Unable to parse the specified CF option " + name);
+ }
+ const auto& opt_info = iter->second;
+ if (opt_info.verification != OptionVerificationType::kDeprecated &&
+ ParseOptionHelper(
+ reinterpret_cast<char*>(new_options) + opt_info.offset,
+ opt_info.type, value)) {
+ return Status::OK();
+ }
+ switch (opt_info.verification) {
+ case OptionVerificationType::kByName:
+ case OptionVerificationType::kByNameAllowNull:
+ case OptionVerificationType::kByNameAllowFromNull:
+ return Status::NotSupported(
+ "Deserializing the specified CF option " + name +
+ " is not supported");
+ case OptionVerificationType::kDeprecated:
+ return Status::OK();
+ default:
+ return Status::InvalidArgument(
+ "Unable to parse the specified CF option " + name);
+ }
+ }
+ } catch (const std::exception&) {
+ return Status::InvalidArgument(
+ "unable to parse the specified option " + name);
+ }
+ return Status::OK();
+}
+
+template <typename T>
+bool SerializeSingleStructOption(
+ std::string* opt_string, const T& options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_info,
+ const std::string& name, const std::string& delimiter) {
+ auto iter = type_info.find(name);
+ if (iter == type_info.end()) {
+ return false;
+ }
+ auto& opt_info = iter->second;
+ const char* opt_address =
+ reinterpret_cast<const char*>(&options) + opt_info.offset;
+ std::string value;
+ bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
+ if (result) {
+ *opt_string = name + "=" + value + delimiter;
+ }
+ return result;
+}
+
+template <typename T>
+Status GetStringFromStruct(
+ std::string* opt_string, const T& options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_info,
+ const std::string& delimiter) {
+ assert(opt_string);
+ opt_string->clear();
+ for (auto iter = type_info.begin(); iter != type_info.end(); ++iter) {
+ if (iter->second.verification == OptionVerificationType::kDeprecated) {
+ // If the option is no longer used in rocksdb and marked as deprecated,
+ // we skip it in the serialization.
+ continue;
+ }
+ std::string single_output;
+ bool result = SerializeSingleStructOption<T>(
+ &single_output, options, type_info, iter->first, delimiter);
+ if (result) {
+ opt_string->append(single_output);
+ } else {
+ return Status::InvalidArgument("failed to serialize %s\n",
+ iter->first.c_str());
+ }
+ assert(result);
+ }
+ return Status::OK();
+}
+
+Status GetStringFromDBOptions(std::string* opt_string,
+ const DBOptions& db_options,
+ const std::string& delimiter) {
+ return GetStringFromStruct<DBOptions>(opt_string, db_options,
+ db_options_type_info, delimiter);
+}
+
+Status GetStringFromColumnFamilyOptions(std::string* opt_string,
+ const ColumnFamilyOptions& cf_options,
+ const std::string& delimiter) {
+ return GetStringFromStruct<ColumnFamilyOptions>(
+ opt_string, cf_options, cf_options_type_info, delimiter);
+}
+
+Status GetStringFromCompressionType(std::string* compression_str,
+ CompressionType compression_type) {
+ bool ok = SerializeEnum<CompressionType>(compression_type_string_map,
+ compression_type, compression_str);
+ if (ok) {
+ return Status::OK();
+ } else {
+ return Status::InvalidArgument("Invalid compression types");
+ }
+}
+
+std::vector<CompressionType> GetSupportedCompressions() {
+ std::vector<CompressionType> supported_compressions;
+ for (const auto& comp_to_name : compression_type_string_map) {
+ CompressionType t = comp_to_name.second;
+ if (t != kDisableCompressionOption && CompressionTypeSupported(t)) {
+ supported_compressions.push_back(t);
+ }
+ }
+ return supported_compressions;
+}
+
+Status ParseDBOption(const std::string& name,
+ const std::string& org_value,
+ DBOptions* new_options,
+ bool input_strings_escaped = false) {
+ const std::string& value =
+ input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
+ try {
+ if (name == "rate_limiter_bytes_per_sec") {
+ new_options->rate_limiter.reset(
+ NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value))));
+ } else if (name == kNameEnv) {
+ // Currently `Env` can be deserialized from object registry only.
+ Env* env = new_options->env;
+ Status status = Env::LoadEnv(value, &env);
+ // Only support static env for now.
+ if (status.ok()) {
+ new_options->env = env;
+ }
+ } else {
+ auto iter = db_options_type_info.find(name);
+ if (iter == db_options_type_info.end()) {
+ return Status::InvalidArgument("Unrecognized option DBOptions:", name);
+ }
+ const auto& opt_info = iter->second;
+ if (opt_info.verification != OptionVerificationType::kDeprecated &&
+ ParseOptionHelper(
+ reinterpret_cast<char*>(new_options) + opt_info.offset,
+ opt_info.type, value)) {
+ return Status::OK();
+ }
+ switch (opt_info.verification) {
+ case OptionVerificationType::kByName:
+ case OptionVerificationType::kByNameAllowNull:
+ return Status::NotSupported(
+ "Deserializing the specified DB option " + name +
+ " is not supported");
+ case OptionVerificationType::kDeprecated:
+ return Status::OK();
+ default:
+ return Status::InvalidArgument(
+ "Unable to parse the specified DB option " + name);
+ }
+ }
+ } catch (const std::exception&) {
+ return Status::InvalidArgument("Unable to parse DBOptions:", name);
+ }
+ return Status::OK();
+}
+
+Status GetColumnFamilyOptionsFromMap(
+ const ColumnFamilyOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ ColumnFamilyOptions* new_options, bool input_strings_escaped,
+ bool ignore_unknown_options) {
+ return GetColumnFamilyOptionsFromMapInternal(
+ base_options, opts_map, new_options, input_strings_escaped, nullptr,
+ ignore_unknown_options);
+}
+
+Status GetColumnFamilyOptionsFromMapInternal(
+ const ColumnFamilyOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ ColumnFamilyOptions* new_options, bool input_strings_escaped,
+ std::vector<std::string>* unsupported_options_names,
+ bool ignore_unknown_options) {
+ assert(new_options);
+ *new_options = base_options;
+ if (unsupported_options_names) {
+ unsupported_options_names->clear();
+ }
+ for (const auto& o : opts_map) {
+ auto s = ParseColumnFamilyOption(o.first, o.second, new_options,
+ input_strings_escaped);
+ if (!s.ok()) {
+ if (s.IsNotSupported()) {
+ // If the deserialization of the specified option is not supported
+ // and an output vector of unsupported_options is provided, then
+ // we log the name of the unsupported option and proceed.
+ if (unsupported_options_names != nullptr) {
+ unsupported_options_names->push_back(o.first);
+ }
+ // Note that we still return Status::OK in such case to maintain
+ // the backward compatibility in the old public API defined in
+ // rocksdb/convenience.h
+ } else if (s.IsInvalidArgument() && ignore_unknown_options) {
+ continue;
+ } else {
+ // Restore "new_options" to the default "base_options".
+ *new_options = base_options;
+ return s;
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status GetColumnFamilyOptionsFromString(
+ const ColumnFamilyOptions& base_options,
+ const std::string& opts_str,
+ ColumnFamilyOptions* new_options) {
+ std::unordered_map<std::string, std::string> opts_map;
+ Status s = StringToMap(opts_str, &opts_map);
+ if (!s.ok()) {
+ *new_options = base_options;
+ return s;
+ }
+ return GetColumnFamilyOptionsFromMap(base_options, opts_map, new_options);
+}
+
+Status GetDBOptionsFromMap(
+ const DBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ DBOptions* new_options, bool input_strings_escaped,
+ bool ignore_unknown_options) {
+ return GetDBOptionsFromMapInternal(base_options, opts_map, new_options,
+ input_strings_escaped, nullptr,
+ ignore_unknown_options);
+}
+
+Status GetDBOptionsFromMapInternal(
+ const DBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ DBOptions* new_options, bool input_strings_escaped,
+ std::vector<std::string>* unsupported_options_names,
+ bool ignore_unknown_options) {
+ assert(new_options);
+ *new_options = base_options;
+ if (unsupported_options_names) {
+ unsupported_options_names->clear();
+ }
+ for (const auto& o : opts_map) {
+ auto s = ParseDBOption(o.first, o.second,
+ new_options, input_strings_escaped);
+ if (!s.ok()) {
+ if (s.IsNotSupported()) {
+ // If the deserialization of the specified option is not supported
+ // and an output vector of unsupported_options is provided, then
+ // we log the name of the unsupported option and proceed.
+ if (unsupported_options_names != nullptr) {
+ unsupported_options_names->push_back(o.first);
+ }
+ // Note that we still return Status::OK in such case to maintain
+ // the backward compatibility in the old public API defined in
+ // rocksdb/convenience.h
+ } else if (s.IsInvalidArgument() && ignore_unknown_options) {
+ continue;
+ } else {
+ // Restore "new_options" to the default "base_options".
+ *new_options = base_options;
+ return s;
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status GetDBOptionsFromString(
+ const DBOptions& base_options,
+ const std::string& opts_str,
+ DBOptions* new_options) {
+ std::unordered_map<std::string, std::string> opts_map;
+ Status s = StringToMap(opts_str, &opts_map);
+ if (!s.ok()) {
+ *new_options = base_options;
+ return s;
+ }
+ return GetDBOptionsFromMap(base_options, opts_map, new_options);
+}
+
+Status GetOptionsFromString(const Options& base_options,
+ const std::string& opts_str, Options* new_options) {
+ std::unordered_map<std::string, std::string> opts_map;
+ Status s = StringToMap(opts_str, &opts_map);
+ if (!s.ok()) {
+ return s;
+ }
+ DBOptions new_db_options(base_options);
+ ColumnFamilyOptions new_cf_options(base_options);
+ for (const auto& o : opts_map) {
+ if (ParseDBOption(o.first, o.second, &new_db_options).ok()) {
+ } else if (ParseColumnFamilyOption(
+ o.first, o.second, &new_cf_options).ok()) {
+ } else {
+ return Status::InvalidArgument("Can't parse option " + o.first);
+ }
+ }
+ *new_options = Options(new_db_options, new_cf_options);
+ return Status::OK();
+}
+
+Status GetTableFactoryFromMap(
+ const std::string& factory_name,
+ const std::unordered_map<std::string, std::string>& opt_map,
+ std::shared_ptr<TableFactory>* table_factory, bool ignore_unknown_options) {
+ Status s;
+ if (factory_name == BlockBasedTableFactory().Name()) {
+ BlockBasedTableOptions bbt_opt;
+ s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map,
+ &bbt_opt,
+ true, /* input_strings_escaped */
+ ignore_unknown_options);
+ if (!s.ok()) {
+ return s;
+ }
+ table_factory->reset(new BlockBasedTableFactory(bbt_opt));
+ return Status::OK();
+ } else if (factory_name == PlainTableFactory().Name()) {
+ PlainTableOptions pt_opt;
+ s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt,
+ true, /* input_strings_escaped */
+ ignore_unknown_options);
+ if (!s.ok()) {
+ return s;
+ }
+ table_factory->reset(new PlainTableFactory(pt_opt));
+ return Status::OK();
+ }
+ // Return OK for not supported table factories as TableFactory
+ // Deserialization is optional.
+ table_factory->reset();
+ return Status::OK();
+}
+
+std::unordered_map<std::string, OptionTypeInfo>
+ OptionsHelper::db_options_type_info = {
+ /*
+ // not yet supported
+ std::shared_ptr<Cache> row_cache;
+ std::shared_ptr<DeleteScheduler> delete_scheduler;
+ std::shared_ptr<Logger> info_log;
+ std::shared_ptr<RateLimiter> rate_limiter;
+ std::shared_ptr<Statistics> statistics;
+ std::vector<DbPath> db_paths;
+ std::vector<std::shared_ptr<EventListener>> listeners;
+ */
+ {"advise_random_on_open",
+ {offsetof(struct DBOptions, advise_random_on_open),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"allow_mmap_reads",
+ {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"allow_fallocate",
+ {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"allow_mmap_writes",
+ {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"use_direct_reads",
+ {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"use_direct_writes",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
+ 0}},
+ {"use_direct_io_for_flush_and_compaction",
+ {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"allow_2pc",
+ {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"allow_os_buffer",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"create_if_missing",
+ {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"create_missing_column_families",
+ {offsetof(struct DBOptions, create_missing_column_families),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"disableDataSync",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
+ 0}},
+ {"disable_data_sync", // for compatibility
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
+ 0}},
+ {"enable_thread_tracking",
+ {offsetof(struct DBOptions, enable_thread_tracking),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"error_if_exists",
+ {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"is_fd_close_on_exec",
+ {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"paranoid_checks",
+ {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"skip_log_error_on_recovery",
+ {offsetof(struct DBOptions, skip_log_error_on_recovery),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"skip_stats_update_on_db_open",
+ {offsetof(struct DBOptions, skip_stats_update_on_db_open),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"skip_checking_sst_file_sizes_on_db_open",
+ {offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"new_table_reader_for_compaction_inputs",
+ {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"compaction_readahead_size",
+ {offsetof(struct DBOptions, compaction_readahead_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, compaction_readahead_size)}},
+ {"random_access_max_buffer_size",
+ {offsetof(struct DBOptions, random_access_max_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
+ {"use_adaptive_mutex",
+ {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"use_fsync",
+ {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"max_background_jobs",
+ {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, max_background_jobs)}},
+ {"max_background_compactions",
+ {offsetof(struct DBOptions, max_background_compactions),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, max_background_compactions)}},
+ {"base_background_compactions",
+ {offsetof(struct DBOptions, base_background_compactions),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, base_background_compactions)}},
+ {"max_background_flushes",
+ {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt,
+ OptionVerificationType::kNormal, false, 0}},
+ {"max_file_opening_threads",
+ {offsetof(struct DBOptions, max_file_opening_threads),
+ OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
+ {"max_open_files",
+ {offsetof(struct DBOptions, max_open_files), OptionType::kInt,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, max_open_files)}},
+ {"table_cache_numshardbits",
+ {offsetof(struct DBOptions, table_cache_numshardbits),
+ OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
+ {"db_write_buffer_size",
+ {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT,
+ OptionVerificationType::kNormal, false, 0}},
+ {"keep_log_file_num",
+ {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT,
+ OptionVerificationType::kNormal, false, 0}},
+ {"recycle_log_file_num",
+ {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT,
+ OptionVerificationType::kNormal, false, 0}},
+ {"log_file_time_to_roll",
+ {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT,
+ OptionVerificationType::kNormal, false, 0}},
+ {"manifest_preallocation_size",
+ {offsetof(struct DBOptions, manifest_preallocation_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
+ {"max_log_file_size",
+ {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT,
+ OptionVerificationType::kNormal, false, 0}},
+ {"db_log_dir",
+ {offsetof(struct DBOptions, db_log_dir), OptionType::kString,
+ OptionVerificationType::kNormal, false, 0}},
+ {"wal_dir",
+ {offsetof(struct DBOptions, wal_dir), OptionType::kString,
+ OptionVerificationType::kNormal, false, 0}},
+ {"max_subcompactions",
+ {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T,
+ OptionVerificationType::kNormal, false, 0}},
+ {"WAL_size_limit_MB",
+ {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, false, 0}},
+ {"WAL_ttl_seconds",
+ {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, false, 0}},
+ {"bytes_per_sync",
+ {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, bytes_per_sync)}},
+ {"delayed_write_rate",
+ {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, delayed_write_rate)}},
+ {"delete_obsolete_files_period_micros",
+ {offsetof(struct DBOptions, delete_obsolete_files_period_micros),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions,
+ delete_obsolete_files_period_micros)}},
+ {"max_manifest_file_size",
+ {offsetof(struct DBOptions, max_manifest_file_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
+ {"max_total_wal_size",
+ {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, max_total_wal_size)}},
+ {"wal_bytes_per_sync",
+ {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, wal_bytes_per_sync)}},
+ {"strict_bytes_per_sync",
+ {offsetof(struct DBOptions, strict_bytes_per_sync),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, strict_bytes_per_sync)}},
+ {"stats_dump_period_sec",
+ {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, stats_dump_period_sec)}},
+ {"stats_persist_period_sec",
+ {offsetof(struct DBOptions, stats_persist_period_sec),
+ OptionType::kUInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, stats_persist_period_sec)}},
+ {"persist_stats_to_disk",
+ {offsetof(struct DBOptions, persist_stats_to_disk),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false,
+ offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}},
+ {"stats_history_buffer_size",
+ {offsetof(struct DBOptions, stats_history_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, stats_history_buffer_size)}},
+ {"fail_if_options_file_error",
+ {offsetof(struct DBOptions, fail_if_options_file_error),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"enable_pipelined_write",
+ {offsetof(struct DBOptions, enable_pipelined_write),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"unordered_write",
+ {offsetof(struct DBOptions, unordered_write), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"allow_concurrent_memtable_write",
+ {offsetof(struct DBOptions, allow_concurrent_memtable_write),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"wal_recovery_mode",
+ {offsetof(struct DBOptions, wal_recovery_mode),
+ OptionType::kWALRecoveryMode, OptionVerificationType::kNormal, false,
+ 0}},
+ {"enable_write_thread_adaptive_yield",
+ {offsetof(struct DBOptions, enable_write_thread_adaptive_yield),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"write_thread_slow_yield_usec",
+ {offsetof(struct DBOptions, write_thread_slow_yield_usec),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
+ {"max_write_batch_group_size_bytes",
+ {offsetof(struct DBOptions, max_write_batch_group_size_bytes),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
+ {"write_thread_max_yield_usec",
+ {offsetof(struct DBOptions, write_thread_max_yield_usec),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
+ {"access_hint_on_compaction_start",
+ {offsetof(struct DBOptions, access_hint_on_compaction_start),
+ OptionType::kAccessHint, OptionVerificationType::kNormal, false, 0}},
+ {"info_log_level",
+ {offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel,
+ OptionVerificationType::kNormal, false, 0}},
+ {"dump_malloc_stats",
+ {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false, 0}},
+ {"avoid_flush_during_recovery",
+ {offsetof(struct DBOptions, avoid_flush_during_recovery),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"avoid_flush_during_shutdown",
+ {offsetof(struct DBOptions, avoid_flush_during_shutdown),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}},
+ {"writable_file_max_buffer_size",
+ {offsetof(struct DBOptions, writable_file_max_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}},
+ {"allow_ingest_behind",
+ {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false,
+ offsetof(struct ImmutableDBOptions, allow_ingest_behind)}},
+ {"preserve_deletes",
+ {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false,
+ offsetof(struct ImmutableDBOptions, preserve_deletes)}},
+ {"concurrent_prepare", // Deprecated by two_write_queues
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
+ 0}},
+ {"two_write_queues",
+ {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false,
+ offsetof(struct ImmutableDBOptions, two_write_queues)}},
+ {"manual_wal_flush",
+ {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false,
+ offsetof(struct ImmutableDBOptions, manual_wal_flush)}},
+ {"seq_per_batch",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
+ 0}},
+ {"atomic_flush",
+ {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean,
+ OptionVerificationType::kNormal, false,
+ offsetof(struct ImmutableDBOptions, atomic_flush)}},
+ {"avoid_unnecessary_blocking_io",
+ {offsetof(struct DBOptions, avoid_unnecessary_blocking_io),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false,
+ offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}},
+ {"write_dbid_to_manifest",
+ {offsetof(struct DBOptions, write_dbid_to_manifest),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"log_readahead_size",
+ {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT,
+ OptionVerificationType::kNormal, false, 0}},
+};
+
+std::unordered_map<std::string, BlockBasedTableOptions::IndexType>
+ OptionsHelper::block_base_table_index_type_string_map = {
+ {"kBinarySearch", BlockBasedTableOptions::IndexType::kBinarySearch},
+ {"kHashSearch", BlockBasedTableOptions::IndexType::kHashSearch},
+ {"kTwoLevelIndexSearch",
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch},
+ {"kBinarySearchWithFirstKey",
+ BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey}};
+
+std::unordered_map<std::string, BlockBasedTableOptions::DataBlockIndexType>
+ OptionsHelper::block_base_table_data_block_index_type_string_map = {
+ {"kDataBlockBinarySearch",
+ BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch},
+ {"kDataBlockBinaryAndHash",
+ BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinaryAndHash}};
+
+std::unordered_map<std::string, BlockBasedTableOptions::IndexShorteningMode>
+ OptionsHelper::block_base_table_index_shortening_mode_string_map = {
+ {"kNoShortening",
+ BlockBasedTableOptions::IndexShorteningMode::kNoShortening},
+ {"kShortenSeparators",
+ BlockBasedTableOptions::IndexShorteningMode::kShortenSeparators},
+ {"kShortenSeparatorsAndSuccessor",
+ BlockBasedTableOptions::IndexShorteningMode::
+ kShortenSeparatorsAndSuccessor}};
+
+std::unordered_map<std::string, EncodingType>
+ OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain},
+ {"kPrefix", kPrefix}};
+
+std::unordered_map<std::string, CompactionStyle>
+ OptionsHelper::compaction_style_string_map = {
+ {"kCompactionStyleLevel", kCompactionStyleLevel},
+ {"kCompactionStyleUniversal", kCompactionStyleUniversal},
+ {"kCompactionStyleFIFO", kCompactionStyleFIFO},
+ {"kCompactionStyleNone", kCompactionStyleNone}};
+
+std::unordered_map<std::string, CompactionPri>
+ OptionsHelper::compaction_pri_string_map = {
+ {"kByCompensatedSize", kByCompensatedSize},
+ {"kOldestLargestSeqFirst", kOldestLargestSeqFirst},
+ {"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst},
+ {"kMinOverlappingRatio", kMinOverlappingRatio}};
+
+std::unordered_map<std::string, WALRecoveryMode>
+ OptionsHelper::wal_recovery_mode_string_map = {
+ {"kTolerateCorruptedTailRecords",
+ WALRecoveryMode::kTolerateCorruptedTailRecords},
+ {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency},
+ {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery},
+ {"kSkipAnyCorruptedRecords",
+ WALRecoveryMode::kSkipAnyCorruptedRecords}};
+
+std::unordered_map<std::string, DBOptions::AccessHint>
+ OptionsHelper::access_hint_string_map = {
+ {"NONE", DBOptions::AccessHint::NONE},
+ {"NORMAL", DBOptions::AccessHint::NORMAL},
+ {"SEQUENTIAL", DBOptions::AccessHint::SEQUENTIAL},
+ {"WILLNEED", DBOptions::AccessHint::WILLNEED}};
+
+std::unordered_map<std::string, InfoLogLevel>
+ OptionsHelper::info_log_level_string_map = {
+ {"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL},
+ {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL},
+ {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL},
+ {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL},
+ {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
+ {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};
+
+ColumnFamilyOptions OptionsHelper::dummy_cf_options;
+CompactionOptionsFIFO OptionsHelper::dummy_comp_options;
+LRUCacheOptions OptionsHelper::dummy_lru_cache_options;
+CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal;
+
+// offset_of is used to get the offset of a class data member
+// ex: offset_of(&ColumnFamilyOptions::num_levels)
+// This call will return the offset of num_levels in ColumnFamilyOptions class
+//
+// This is the same as offsetof() but allow us to work with non standard-layout
+// classes and structures
+// refs:
+// http://en.cppreference.com/w/cpp/concept/StandardLayoutType
+// https://gist.github.com/graphitemaster/494f21190bb2c63c5516
+template <typename T1>
+int offset_of(T1 ColumnFamilyOptions::*member) {
+ return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) -
+ size_t(&OptionsHelper::dummy_cf_options));
+}
+template <typename T1>
+int offset_of(T1 AdvancedColumnFamilyOptions::*member) {
+ return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) -
+ size_t(&OptionsHelper::dummy_cf_options));
+}
+template <typename T1>
+int offset_of(T1 CompactionOptionsFIFO::*member) {
+ return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) -
+ size_t(&OptionsHelper::dummy_comp_options));
+}
+template <typename T1>
+int offset_of(T1 LRUCacheOptions::*member) {
+ return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) -
+ size_t(&OptionsHelper::dummy_lru_cache_options));
+}
+template <typename T1>
+int offset_of(T1 CompactionOptionsUniversal::*member) {
+ return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) -
+ size_t(&OptionsHelper::dummy_comp_options_universal));
+}
+
+std::unordered_map<std::string, OptionTypeInfo>
+ OptionsHelper::cf_options_type_info = {
+ /* not yet supported
+ CompressionOptions compression_opts;
+ TablePropertiesCollectorFactories table_properties_collector_factories;
+ typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
+ TablePropertiesCollectorFactories;
+ UpdateStatus (*inplace_callback)(char* existing_value,
+ uint34_t* existing_value_size,
+ Slice delta_value,
+ std::string* merged_value);
+ std::vector<DbPath> cf_paths;
+ */
+ {"report_bg_io_stats",
+ {offset_of(&ColumnFamilyOptions::report_bg_io_stats),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, report_bg_io_stats)}},
+ {"compaction_measure_io_stats",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
+ 0}},
+ {"disable_auto_compactions",
+ {offset_of(&ColumnFamilyOptions::disable_auto_compactions),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, disable_auto_compactions)}},
+ {"filter_deletes",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"inplace_update_support",
+ {offset_of(&ColumnFamilyOptions::inplace_update_support),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"level_compaction_dynamic_level_bytes",
+ {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"optimize_filters_for_hits",
+ {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"paranoid_file_checks",
+ {offset_of(&ColumnFamilyOptions::paranoid_file_checks),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, paranoid_file_checks)}},
+ {"force_consistency_checks",
+ {offset_of(&ColumnFamilyOptions::force_consistency_checks),
+ OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
+ {"purge_redundant_kvs_while_flush",
+ {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush),
+ OptionType::kBoolean, OptionVerificationType::kDeprecated, false, 0}},
+ {"verify_checksums_in_compaction",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"soft_pending_compaction_bytes_limit",
+ {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions,
+ soft_pending_compaction_bytes_limit)}},
+ {"hard_pending_compaction_bytes_limit",
+ {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions,
+ hard_pending_compaction_bytes_limit)}},
+ {"hard_rate_limit",
+ {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"soft_rate_limit",
+ {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"max_compaction_bytes",
+ {offset_of(&ColumnFamilyOptions::max_compaction_bytes),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, max_compaction_bytes)}},
+ {"expanded_compaction_factor",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}},
+ {"level0_file_num_compaction_trigger",
+ {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions,
+ level0_file_num_compaction_trigger)}},
+ {"level0_slowdown_writes_trigger",
+ {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}},
+ {"level0_stop_writes_trigger",
+ {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}},
+ {"max_grandparent_overlap_factor",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}},
+ {"max_mem_compaction_level",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated, false, 0}},
+ {"max_write_buffer_number",
+ {offset_of(&ColumnFamilyOptions::max_write_buffer_number),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, max_write_buffer_number)}},
+ {"max_write_buffer_number_to_maintain",
+ {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain),
+ OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
+ {"max_write_buffer_size_to_maintain",
+ {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain),
+ OptionType::kInt64T, OptionVerificationType::kNormal, false, 0}},
+ {"min_write_buffer_number_to_merge",
+ {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge),
+ OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
+ {"num_levels",
+ {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt,
+ OptionVerificationType::kNormal, false, 0}},
+ {"source_compaction_factor",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}},
+ {"target_file_size_multiplier",
+ {offset_of(&ColumnFamilyOptions::target_file_size_multiplier),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, target_file_size_multiplier)}},
+ {"arena_block_size",
+ {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, arena_block_size)}},
+ {"inplace_update_num_locks",
+ {offset_of(&ColumnFamilyOptions::inplace_update_num_locks),
+ OptionType::kSizeT, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, inplace_update_num_locks)}},
+ {"max_successive_merges",
+ {offset_of(&ColumnFamilyOptions::max_successive_merges),
+ OptionType::kSizeT, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, max_successive_merges)}},
+ {"memtable_huge_page_size",
+ {offset_of(&ColumnFamilyOptions::memtable_huge_page_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, memtable_huge_page_size)}},
+ {"memtable_prefix_bloom_huge_page_tlb_size",
+ {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, true, 0}},
+ {"write_buffer_size",
+ {offset_of(&ColumnFamilyOptions::write_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, write_buffer_size)}},
+ {"bloom_locality",
+ {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T,
+ OptionVerificationType::kNormal, false, 0}},
+ {"memtable_prefix_bloom_bits",
+ {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"memtable_prefix_bloom_size_ratio",
+ {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio),
+ OptionType::kDouble, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}},
+ {"memtable_prefix_bloom_probes",
+ {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"memtable_whole_key_filtering",
+ {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}},
+ {"min_partial_merge_operands",
+ {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"max_bytes_for_level_base",
+ {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, max_bytes_for_level_base)}},
+ {"snap_refresh_nanos",
+ {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, true,
+ 0}},
+ {"max_bytes_for_level_multiplier",
+ {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier),
+ OptionType::kDouble, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}},
+ {"max_bytes_for_level_multiplier_additional",
+ {offset_of(
+ &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional),
+ OptionType::kVectorInt, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions,
+ max_bytes_for_level_multiplier_additional)}},
+ {"max_sequential_skip_in_iterations",
+ {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions,
+ max_sequential_skip_in_iterations)}},
+ {"target_file_size_base",
+ {offset_of(&ColumnFamilyOptions::target_file_size_base),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, target_file_size_base)}},
+ {"rate_limit_delay_max_milliseconds",
+ {0, OptionType::kUInt, OptionVerificationType::kDeprecated, false, 0}},
+ {"compression",
+ {offset_of(&ColumnFamilyOptions::compression),
+ OptionType::kCompressionType, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, compression)}},
+ {"compression_per_level",
+ {offset_of(&ColumnFamilyOptions::compression_per_level),
+ OptionType::kVectorCompressionType, OptionVerificationType::kNormal,
+ false, 0}},
+ {"bottommost_compression",
+ {offset_of(&ColumnFamilyOptions::bottommost_compression),
+ OptionType::kCompressionType, OptionVerificationType::kNormal, false,
+ 0}},
+ {kNameComparator,
+ {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator,
+ OptionVerificationType::kByName, false, 0}},
+ {"prefix_extractor",
+ {offset_of(&ColumnFamilyOptions::prefix_extractor),
+ OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
+ true, offsetof(struct MutableCFOptions, prefix_extractor)}},
+ {"memtable_insert_with_hint_prefix_extractor",
+ {offset_of(
+ &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor),
+ OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
+ false, 0}},
+ {"memtable_factory",
+ {offset_of(&ColumnFamilyOptions::memtable_factory),
+ OptionType::kMemTableRepFactory, OptionVerificationType::kByName,
+ false, 0}},
+ {"table_factory",
+ {offset_of(&ColumnFamilyOptions::table_factory),
+ OptionType::kTableFactory, OptionVerificationType::kByName, false,
+ 0}},
+ {"compaction_filter",
+ {offset_of(&ColumnFamilyOptions::compaction_filter),
+ OptionType::kCompactionFilter, OptionVerificationType::kByName, false,
+ 0}},
+ {"compaction_filter_factory",
+ {offset_of(&ColumnFamilyOptions::compaction_filter_factory),
+ OptionType::kCompactionFilterFactory, OptionVerificationType::kByName,
+ false, 0}},
+ {kNameMergeOperator,
+ {offset_of(&ColumnFamilyOptions::merge_operator),
+ OptionType::kMergeOperator,
+ OptionVerificationType::kByNameAllowFromNull, false, 0}},
+ {"compaction_style",
+ {offset_of(&ColumnFamilyOptions::compaction_style),
+ OptionType::kCompactionStyle, OptionVerificationType::kNormal, false,
+ 0}},
+ {"compaction_pri",
+ {offset_of(&ColumnFamilyOptions::compaction_pri),
+ OptionType::kCompactionPri, OptionVerificationType::kNormal, false,
+ 0}},
+ {"compaction_options_fifo",
+ {offset_of(&ColumnFamilyOptions::compaction_options_fifo),
+ OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal,
+ true, offsetof(struct MutableCFOptions, compaction_options_fifo)}},
+ {"compaction_options_universal",
+ {offset_of(&ColumnFamilyOptions::compaction_options_universal),
+ OptionType::kCompactionOptionsUniversal,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, compaction_options_universal)}},
+ {"ttl",
+ {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, ttl)}},
+ {"periodic_compaction_seconds",
+ {offset_of(&ColumnFamilyOptions::periodic_compaction_seconds),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, periodic_compaction_seconds)}},
+ {"sample_for_compression",
+ {offset_of(&ColumnFamilyOptions::sample_for_compression),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct MutableCFOptions, sample_for_compression)}}};
+
+std::unordered_map<std::string, OptionTypeInfo>
+ OptionsHelper::fifo_compaction_options_type_info = {
+ {"max_table_files_size",
+ {offset_of(&CompactionOptionsFIFO::max_table_files_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+ offsetof(struct CompactionOptionsFIFO, max_table_files_size)}},
+ {"ttl",
+ {0, OptionType::kUInt64T,
+ OptionVerificationType::kDeprecated, false,
+ 0}},
+ {"allow_compaction",
+ {offset_of(&CompactionOptionsFIFO::allow_compaction),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct CompactionOptionsFIFO, allow_compaction)}}};
+
+std::unordered_map<std::string, OptionTypeInfo>
+ OptionsHelper::universal_compaction_options_type_info = {
+ {"size_ratio",
+ {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt,
+ OptionVerificationType::kNormal, true,
+ offsetof(class CompactionOptionsUniversal, size_ratio)}},
+ {"min_merge_width",
+ {offset_of(&CompactionOptionsUniversal::min_merge_width),
+ OptionType::kUInt, OptionVerificationType::kNormal, true,
+ offsetof(class CompactionOptionsUniversal, min_merge_width)}},
+ {"max_merge_width",
+ {offset_of(&CompactionOptionsUniversal::max_merge_width),
+ OptionType::kUInt, OptionVerificationType::kNormal, true,
+ offsetof(class CompactionOptionsUniversal, max_merge_width)}},
+ {"max_size_amplification_percent",
+ {offset_of(
+ &CompactionOptionsUniversal::max_size_amplification_percent),
+ OptionType::kUInt, OptionVerificationType::kNormal, true,
+ offsetof(class CompactionOptionsUniversal,
+ max_size_amplification_percent)}},
+ {"compression_size_percent",
+ {offset_of(&CompactionOptionsUniversal::compression_size_percent),
+ OptionType::kInt, OptionVerificationType::kNormal, true,
+ offsetof(class CompactionOptionsUniversal,
+ compression_size_percent)}},
+ {"stop_style",
+ {offset_of(&CompactionOptionsUniversal::stop_style),
+ OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
+ true, offsetof(class CompactionOptionsUniversal, stop_style)}},
+ {"allow_trivial_move",
+ {offset_of(&CompactionOptionsUniversal::allow_trivial_move),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}};
+
+std::unordered_map<std::string, CompactionStopStyle>
+ OptionsHelper::compaction_stop_style_string_map = {
+ {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
+ {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
+
+std::unordered_map<std::string, OptionTypeInfo>
+ OptionsHelper::lru_cache_options_type_info = {
+ {"capacity",
+ {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct LRUCacheOptions, capacity)}},
+ {"num_shard_bits",
+ {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct LRUCacheOptions, num_shard_bits)}},
+ {"strict_capacity_limit",
+ {offset_of(&LRUCacheOptions::strict_capacity_limit),
+ OptionType::kBoolean, OptionVerificationType::kNormal, true,
+ offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
+ {"high_pri_pool_ratio",
+ {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble,
+ OptionVerificationType::kNormal, true,
+ offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}};
+
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_helper.h b/src/rocksdb/options/options_helper.h
new file mode 100644
index 000000000..c75fdb1ce
--- /dev/null
+++ b/src/rocksdb/options/options_helper.h
@@ -0,0 +1,233 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <map>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "options/cf_options.h"
+#include "options/db_options.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table.h"
+#include "rocksdb/universal_compaction.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
+ const MutableDBOptions& mutable_db_options);
+
+ColumnFamilyOptions BuildColumnFamilyOptions(
+ const ColumnFamilyOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options);
+
+#ifndef ROCKSDB_LITE
+
+Status GetMutableOptionsFromStrings(
+ const MutableCFOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ Logger* info_log, MutableCFOptions* new_options);
+
+Status GetMutableDBOptionsFromStrings(
+ const MutableDBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ MutableDBOptions* new_options);
+
+Status GetTableFactoryFromMap(
+ const std::string& factory_name,
+ const std::unordered_map<std::string, std::string>& opt_map,
+ std::shared_ptr<TableFactory>* table_factory,
+ bool ignore_unknown_options = false);
+
+enum class OptionType {
+ kBoolean,
+ kInt,
+ kInt32T,
+ kInt64T,
+ kVectorInt,
+ kUInt,
+ kUInt32T,
+ kUInt64T,
+ kSizeT,
+ kString,
+ kDouble,
+ kCompactionStyle,
+ kCompactionPri,
+ kSliceTransform,
+ kCompressionType,
+ kVectorCompressionType,
+ kTableFactory,
+ kComparator,
+ kCompactionFilter,
+ kCompactionFilterFactory,
+ kCompactionOptionsFIFO,
+ kCompactionOptionsUniversal,
+ kCompactionStopStyle,
+ kMergeOperator,
+ kMemTableRepFactory,
+ kBlockBasedTableIndexType,
+ kBlockBasedTableDataBlockIndexType,
+ kBlockBasedTableIndexShorteningMode,
+ kFilterPolicy,
+ kFlushBlockPolicyFactory,
+ kChecksumType,
+ kEncodingType,
+ kWALRecoveryMode,
+ kAccessHint,
+ kInfoLogLevel,
+ kLRUCacheOptions,
+ kEnv,
+ kUnknown,
+};
+
+enum class OptionVerificationType {
+ kNormal,
+ kByName, // The option is pointer typed so we can only verify
+ // based on it's name.
+ kByNameAllowNull, // Same as kByName, but it also allows the case
+ // where one of them is a nullptr.
+ kByNameAllowFromNull, // Same as kByName, but it also allows the case
+ // where the old option is nullptr.
+ kDeprecated // The option is no longer used in rocksdb. The RocksDB
+ // OptionsParser will still accept this option if it
+ // happen to exists in some Options file. However,
+ // the parser will not include it in serialization
+ // and verification processes.
+};
+
+// A struct for storing constant option information such as option name,
+// option type, and offset.
+struct OptionTypeInfo {
+ int offset;
+ OptionType type;
+ OptionVerificationType verification;
+ bool is_mutable;
+ int mutable_offset;
+};
+
+// A helper function that converts "opt_address" to a std::string
+// based on the specified OptionType.
+bool SerializeSingleOptionHelper(const char* opt_address,
+ const OptionType opt_type, std::string* value);
+
+// In addition to its public version defined in rocksdb/convenience.h,
+// this further takes an optional output vector "unsupported_options_names",
+// which stores the name of all the unsupported options specified in "opts_map".
+Status GetDBOptionsFromMapInternal(
+ const DBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ DBOptions* new_options, bool input_strings_escaped,
+ std::vector<std::string>* unsupported_options_names = nullptr,
+ bool ignore_unknown_options = false);
+
+// In addition to its public version defined in rocksdb/convenience.h,
+// this further takes an optional output vector "unsupported_options_names",
+// which stores the name of all the unsupported options specified in "opts_map".
+Status GetColumnFamilyOptionsFromMapInternal(
+ const ColumnFamilyOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ ColumnFamilyOptions* new_options, bool input_strings_escaped,
+ std::vector<std::string>* unsupported_options_names = nullptr,
+ bool ignore_unknown_options = false);
+
+bool ParseSliceTransform(
+ const std::string& value,
+ std::shared_ptr<const SliceTransform>* slice_transform);
+
+extern Status StringToMap(
+ const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map);
+
+extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
+ const std::string& value);
+#endif // !ROCKSDB_LITE
+
+struct OptionsHelper {
+ static std::map<CompactionStyle, std::string> compaction_style_to_string;
+ static std::map<CompactionPri, std::string> compaction_pri_to_string;
+ static std::map<CompactionStopStyle, std::string>
+ compaction_stop_style_to_string;
+ static std::unordered_map<std::string, ChecksumType> checksum_type_string_map;
+ static std::unordered_map<std::string, CompressionType>
+ compression_type_string_map;
+#ifndef ROCKSDB_LITE
+ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info;
+ static std::unordered_map<std::string, OptionTypeInfo>
+ fifo_compaction_options_type_info;
+ static std::unordered_map<std::string, OptionTypeInfo>
+ universal_compaction_options_type_info;
+ static std::unordered_map<std::string, CompactionStopStyle>
+ compaction_stop_style_string_map;
+ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info;
+ static std::unordered_map<std::string, OptionTypeInfo>
+ lru_cache_options_type_info;
+ static std::unordered_map<std::string, BlockBasedTableOptions::IndexType>
+ block_base_table_index_type_string_map;
+ static std::unordered_map<std::string,
+ BlockBasedTableOptions::DataBlockIndexType>
+ block_base_table_data_block_index_type_string_map;
+ static std::unordered_map<std::string,
+ BlockBasedTableOptions::IndexShorteningMode>
+ block_base_table_index_shortening_mode_string_map;
+ static std::unordered_map<std::string, EncodingType> encoding_type_string_map;
+ static std::unordered_map<std::string, CompactionStyle>
+ compaction_style_string_map;
+ static std::unordered_map<std::string, CompactionPri>
+ compaction_pri_string_map;
+ static std::unordered_map<std::string, WALRecoveryMode>
+ wal_recovery_mode_string_map;
+ static std::unordered_map<std::string, DBOptions::AccessHint>
+ access_hint_string_map;
+ static std::unordered_map<std::string, InfoLogLevel>
+ info_log_level_string_map;
+ static ColumnFamilyOptions dummy_cf_options;
+ static CompactionOptionsFIFO dummy_comp_options;
+ static LRUCacheOptions dummy_lru_cache_options;
+ static CompactionOptionsUniversal dummy_comp_options_universal;
+#endif // !ROCKSDB_LITE
+};
+
+// Some aliasing
+static auto& compaction_style_to_string =
+ OptionsHelper::compaction_style_to_string;
+static auto& compaction_pri_to_string = OptionsHelper::compaction_pri_to_string;
+static auto& compaction_stop_style_to_string =
+ OptionsHelper::compaction_stop_style_to_string;
+static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map;
+#ifndef ROCKSDB_LITE
+static auto& cf_options_type_info = OptionsHelper::cf_options_type_info;
+static auto& fifo_compaction_options_type_info =
+ OptionsHelper::fifo_compaction_options_type_info;
+static auto& universal_compaction_options_type_info =
+ OptionsHelper::universal_compaction_options_type_info;
+static auto& compaction_stop_style_string_map =
+ OptionsHelper::compaction_stop_style_string_map;
+static auto& db_options_type_info = OptionsHelper::db_options_type_info;
+static auto& lru_cache_options_type_info =
+ OptionsHelper::lru_cache_options_type_info;
+static auto& compression_type_string_map =
+ OptionsHelper::compression_type_string_map;
+static auto& block_base_table_index_type_string_map =
+ OptionsHelper::block_base_table_index_type_string_map;
+static auto& block_base_table_data_block_index_type_string_map =
+ OptionsHelper::block_base_table_data_block_index_type_string_map;
+static auto& block_base_table_index_shortening_mode_string_map =
+ OptionsHelper::block_base_table_index_shortening_mode_string_map;
+static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map;
+static auto& compaction_style_string_map =
+ OptionsHelper::compaction_style_string_map;
+static auto& compaction_pri_string_map =
+ OptionsHelper::compaction_pri_string_map;
+static auto& wal_recovery_mode_string_map =
+ OptionsHelper::wal_recovery_mode_string_map;
+static auto& access_hint_string_map = OptionsHelper::access_hint_string_map;
+static auto& info_log_level_string_map =
+ OptionsHelper::info_log_level_string_map;
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_parser.cc b/src/rocksdb/options/options_parser.cc
new file mode 100644
index 000000000..e10f932c9
--- /dev/null
+++ b/src/rocksdb/options/options_parser.cc
@@ -0,0 +1,839 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include "options/options_parser.h"
+
+#include <cmath>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "file/read_write_util.h"
+#include "file/writable_file_writer.h"
+#include "options/options_helper.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/db.h"
+#include "test_util/sync_point.h"
+#include "util/cast_util.h"
+#include "util/string_util.h"
+
+#include "port/port.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+static const std::string option_file_header =
+ "# This is a RocksDB option file.\n"
+ "#\n"
+ "# For detailed file format spec, please refer to the example file\n"
+ "# in examples/rocksdb_option_file_example.ini\n"
+ "#\n"
+ "\n";
+
+Status PersistRocksDBOptions(const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs) {
+ TEST_SYNC_POINT("PersistRocksDBOptions:start");
+ if (cf_names.size() != cf_opts.size()) {
+ return Status::InvalidArgument(
+ "cf_names.size() and cf_opts.size() must be the same");
+ }
+ std::unique_ptr<FSWritableFile> wf;
+
+ Status s =
+ fs->NewWritableFile(file_name, FileOptions(), &wf, nullptr);
+ if (!s.ok()) {
+ return s;
+ }
+ std::unique_ptr<WritableFileWriter> writable;
+ writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(),
+ nullptr /* statistics */));
+
+ std::string options_file_content;
+
+ writable->Append(option_file_header + "[" +
+ opt_section_titles[kOptionSectionVersion] +
+ "]\n"
+ " rocksdb_version=" +
+ ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) +
+ "." + ToString(ROCKSDB_PATCH) + "\n");
+ writable->Append(" options_file_version=" +
+ ToString(ROCKSDB_OPTION_FILE_MAJOR) + "." +
+ ToString(ROCKSDB_OPTION_FILE_MINOR) + "\n");
+ writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] +
+ "]\n ");
+
+ s = GetStringFromDBOptions(&options_file_content, db_opt, "\n ");
+ if (!s.ok()) {
+ writable->Close();
+ return s;
+ }
+ writable->Append(options_file_content + "\n");
+
+ for (size_t i = 0; i < cf_opts.size(); ++i) {
+ // CFOptions section
+ writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] +
+ " \"" + EscapeOptionString(cf_names[i]) + "\"]\n ");
+ s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i],
+ "\n ");
+ if (!s.ok()) {
+ writable->Close();
+ return s;
+ }
+ writable->Append(options_file_content + "\n");
+ // TableOptions section
+ auto* tf = cf_opts[i].table_factory.get();
+ if (tf != nullptr) {
+ writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] +
+ tf->Name() + " \"" + EscapeOptionString(cf_names[i]) +
+ "\"]\n ");
+ options_file_content.clear();
+ s = tf->GetOptionString(&options_file_content, "\n ");
+ if (!s.ok()) {
+ return s;
+ }
+ writable->Append(options_file_content + "\n");
+ }
+ }
+ writable->Sync(true /* use_fsync */);
+ writable->Close();
+
+ return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ db_opt, cf_names, cf_opts, file_name, fs);
+}
+
+RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }
+
+void RocksDBOptionsParser::Reset() {
+ db_opt_ = DBOptions();
+ db_opt_map_.clear();
+ cf_names_.clear();
+ cf_opts_.clear();
+ cf_opt_maps_.clear();
+ has_version_section_ = false;
+ has_db_options_ = false;
+ has_default_cf_options_ = false;
+ for (int i = 0; i < 3; ++i) {
+ db_version[i] = 0;
+ opt_file_version[i] = 0;
+ }
+}
+
+bool RocksDBOptionsParser::IsSection(const std::string& line) {
+ if (line.size() < 2) {
+ return false;
+ }
+ if (line[0] != '[' || line[line.size() - 1] != ']') {
+ return false;
+ }
+ return true;
+}
+
+Status RocksDBOptionsParser::ParseSection(OptionSection* section,
+ std::string* title,
+ std::string* argument,
+ const std::string& line,
+ const int line_num) {
+ *section = kOptionSectionUnknown;
+ // A section is of the form [<SectionName> "<SectionArg>"], where
+ // "<SectionArg>" is optional.
+ size_t arg_start_pos = line.find("\"");
+ size_t arg_end_pos = line.rfind("\"");
+ // The following if-then check tries to identify whether the input
+ // section has the optional section argument.
+ if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) {
+ *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true);
+ *argument = UnescapeOptionString(
+ line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1));
+ } else {
+ *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true);
+ *argument = "";
+ }
+ for (int i = 0; i < kOptionSectionUnknown; ++i) {
+ if (title->find(opt_section_titles[i]) == 0) {
+ if (i == kOptionSectionVersion || i == kOptionSectionDBOptions ||
+ i == kOptionSectionCFOptions) {
+ if (title->size() == opt_section_titles[i].size()) {
+ // if true, then it indicats equal
+ *section = static_cast<OptionSection>(i);
+ return CheckSection(*section, *argument, line_num);
+ }
+ } else if (i == kOptionSectionTableOptions) {
+ // This type of sections has a sufffix at the end of the
+ // section title
+ if (title->size() > opt_section_titles[i].size()) {
+ *section = static_cast<OptionSection>(i);
+ return CheckSection(*section, *argument, line_num);
+ }
+ }
+ }
+ }
+ return Status::InvalidArgument(std::string("Unknown section ") + line);
+}
+
+Status RocksDBOptionsParser::InvalidArgument(const int line_num,
+ const std::string& message) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error] ",
+ message + " (at line " + ToString(line_num) + ")");
+}
+
+Status RocksDBOptionsParser::ParseStatement(std::string* name,
+ std::string* value,
+ const std::string& line,
+ const int line_num) {
+ size_t eq_pos = line.find("=");
+ if (eq_pos == std::string::npos) {
+ return InvalidArgument(line_num, "A valid statement must have a '='.");
+ }
+
+ *name = TrimAndRemoveComment(line.substr(0, eq_pos), true);
+ *value =
+ TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1));
+ if (name->empty()) {
+ return InvalidArgument(line_num,
+ "A valid statement must have a variable name.");
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
+ bool ignore_unknown_options,
+ size_t file_readahead_size) {
+ Reset();
+
+ std::unique_ptr<FSSequentialFile> seq_file;
+ Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file,
+ nullptr);
+ if (!s.ok()) {
+ return s;
+ }
+
+ SequentialFileReader sf_reader(std::move(seq_file), file_name,
+ file_readahead_size);
+
+ OptionSection section = kOptionSectionUnknown;
+ std::string title;
+ std::string argument;
+ std::unordered_map<std::string, std::string> opt_map;
+ std::istringstream iss;
+ std::string line;
+ bool has_data = true;
+ // we only support single-lined statement.
+ for (int line_num = 1; ReadOneLine(&iss, &sf_reader, &line, &has_data, &s);
+ ++line_num) {
+ if (!s.ok()) {
+ return s;
+ }
+ line = TrimAndRemoveComment(line);
+ if (line.empty()) {
+ continue;
+ }
+ if (IsSection(line)) {
+ s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
+ opt_map.clear();
+ if (!s.ok()) {
+ return s;
+ }
+
+ // If the option file is not generated by a higher minor version,
+ // there shouldn't be any unknown option.
+ if (ignore_unknown_options && section == kOptionSectionVersion) {
+ if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR &&
+ db_version[1] <= ROCKSDB_MINOR)) {
+ ignore_unknown_options = false;
+ }
+ }
+
+ s = ParseSection(&section, &title, &argument, line, line_num);
+ if (!s.ok()) {
+ return s;
+ }
+ } else {
+ std::string name;
+ std::string value;
+ s = ParseStatement(&name, &value, line, line_num);
+ if (!s.ok()) {
+ return s;
+ }
+ opt_map.insert({name, value});
+ }
+ }
+
+ s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
+ opt_map.clear();
+ if (!s.ok()) {
+ return s;
+ }
+ return ValidityCheck();
+}
+
+Status RocksDBOptionsParser::CheckSection(const OptionSection section,
+ const std::string& section_arg,
+ const int line_num) {
+ if (section == kOptionSectionDBOptions) {
+ if (has_db_options_) {
+ return InvalidArgument(
+ line_num,
+ "More than one DBOption section found in the option config file");
+ }
+ has_db_options_ = true;
+ } else if (section == kOptionSectionCFOptions) {
+ bool is_default_cf = (section_arg == kDefaultColumnFamilyName);
+ if (cf_opts_.size() == 0 && !is_default_cf) {
+ return InvalidArgument(
+ line_num,
+ "Default column family must be the first CFOptions section "
+ "in the option config file");
+ } else if (cf_opts_.size() != 0 && is_default_cf) {
+ return InvalidArgument(
+ line_num,
+ "Default column family must be the first CFOptions section "
+ "in the optio/n config file");
+ } else if (GetCFOptions(section_arg) != nullptr) {
+ return InvalidArgument(
+ line_num,
+ "Two identical column families found in option config file");
+ }
+ has_default_cf_options_ |= is_default_cf;
+ } else if (section == kOptionSectionTableOptions) {
+ if (GetCFOptions(section_arg) == nullptr) {
+ return InvalidArgument(
+ line_num, std::string(
+ "Does not find a matched column family name in "
+ "TableOptions section. Column Family Name:") +
+ section_arg);
+ }
+ } else if (section == kOptionSectionVersion) {
+ if (has_version_section_) {
+ return InvalidArgument(
+ line_num,
+ "More than one Version section found in the option config file.");
+ }
+ has_version_section_ = true;
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
+ const std::string& ver_string,
+ const int max_count,
+ int* version) {
+ int version_index = 0;
+ int current_number = 0;
+ int current_digit_count = 0;
+ bool has_dot = false;
+ for (int i = 0; i < max_count; ++i) {
+ version[i] = 0;
+ }
+ constexpr int kBufferSize = 200;
+ char buffer[kBufferSize];
+ for (size_t i = 0; i < ver_string.size(); ++i) {
+ if (ver_string[i] == '.') {
+ if (version_index >= max_count - 1) {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s can only contains at most %d dots.",
+ ver_name.c_str(), max_count - 1);
+ return Status::InvalidArgument(buffer);
+ }
+ if (current_digit_count == 0) {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s must have at least one digit before each dot.",
+ ver_name.c_str());
+ return Status::InvalidArgument(buffer);
+ }
+ version[version_index++] = current_number;
+ current_number = 0;
+ current_digit_count = 0;
+ has_dot = true;
+ } else if (isdigit(ver_string[i])) {
+ current_number = current_number * 10 + (ver_string[i] - '0');
+ current_digit_count++;
+ } else {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s can only contains dots and numbers.",
+ ver_name.c_str());
+ return Status::InvalidArgument(buffer);
+ }
+ }
+ version[version_index] = current_number;
+ if (has_dot && current_digit_count == 0) {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s must have at least one digit after each dot.",
+ ver_name.c_str());
+ return Status::InvalidArgument(buffer);
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::EndSection(
+ const OptionSection section, const std::string& section_title,
+ const std::string& section_arg,
+ const std::unordered_map<std::string, std::string>& opt_map,
+ bool ignore_unknown_options) {
+ Status s;
+ if (section == kOptionSectionDBOptions) {
+ s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true,
+ ignore_unknown_options);
+ if (!s.ok()) {
+ return s;
+ }
+ db_opt_map_ = opt_map;
+ } else if (section == kOptionSectionCFOptions) {
+ // This condition should be ensured earlier in ParseSection
+ // so we make an assertion here.
+ assert(GetCFOptions(section_arg) == nullptr);
+ cf_names_.emplace_back(section_arg);
+ cf_opts_.emplace_back();
+ s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map,
+ &cf_opts_.back(), true,
+ ignore_unknown_options);
+ if (!s.ok()) {
+ return s;
+ }
+ // keep the parsed string.
+ cf_opt_maps_.emplace_back(opt_map);
+ } else if (section == kOptionSectionTableOptions) {
+ assert(GetCFOptions(section_arg) != nullptr);
+ auto* cf_opt = GetCFOptionsImpl(section_arg);
+ if (cf_opt == nullptr) {
+ return Status::InvalidArgument(
+ "The specified column family must be defined before the "
+ "TableOptions section:",
+ section_arg);
+ }
+ // Ignore error as table factory deserialization is optional
+ s = GetTableFactoryFromMap(
+ section_title.substr(
+ opt_section_titles[kOptionSectionTableOptions].size()),
+ opt_map, &(cf_opt->table_factory), ignore_unknown_options);
+ if (!s.ok()) {
+ return s;
+ }
+ } else if (section == kOptionSectionVersion) {
+ for (const auto pair : opt_map) {
+ if (pair.first == "rocksdb_version") {
+ s = ParseVersionNumber(pair.first, pair.second, 3, db_version);
+ if (!s.ok()) {
+ return s;
+ }
+ } else if (pair.first == "options_file_version") {
+ s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version);
+ if (!s.ok()) {
+ return s;
+ }
+ if (opt_file_version[0] < 1) {
+ return Status::InvalidArgument(
+ "A valid options_file_version must be at least 1.");
+ }
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::ValidityCheck() {
+ if (!has_db_options_) {
+ return Status::Corruption(
+ "A RocksDB Option file must have a single DBOptions section");
+ }
+ if (!has_default_cf_options_) {
+ return Status::Corruption(
+ "A RocksDB Option file must have a single CFOptions:default section");
+ }
+
+ return Status::OK();
+}
+
+std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line,
+ bool trim_only) {
+ size_t start = 0;
+ size_t end = line.size();
+
+ // we only support "#" style comment
+ if (!trim_only) {
+ size_t search_pos = 0;
+ while (search_pos < line.size()) {
+ size_t comment_pos = line.find('#', search_pos);
+ if (comment_pos == std::string::npos) {
+ break;
+ }
+ if (comment_pos == 0 || line[comment_pos - 1] != '\\') {
+ end = comment_pos;
+ break;
+ }
+ search_pos = comment_pos + 1;
+ }
+ }
+
+ while (start < end && isspace(line[start]) != 0) {
+ ++start;
+ }
+
+ // start < end implies end > 0.
+ while (start < end && isspace(line[end - 1]) != 0) {
+ --end;
+ }
+
+ if (start < end) {
+ return line.substr(start, end - start);
+ }
+
+ return "";
+}
+
+namespace {
+bool AreEqualDoubles(const double a, const double b) {
+ return (fabs(a - b) < 0.00001);
+}
+} // namespace
+
+bool AreEqualOptions(
+ const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
+ const std::string& opt_name,
+ const std::unordered_map<std::string, std::string>* opt_map) {
+ const char* offset1 = opt1 + type_info.offset;
+ const char* offset2 = opt2 + type_info.offset;
+
+ switch (type_info.type) {
+ case OptionType::kBoolean:
+ return (*reinterpret_cast<const bool*>(offset1) ==
+ *reinterpret_cast<const bool*>(offset2));
+ case OptionType::kInt:
+ return (*reinterpret_cast<const int*>(offset1) ==
+ *reinterpret_cast<const int*>(offset2));
+ case OptionType::kInt32T:
+ return (*reinterpret_cast<const int32_t*>(offset1) ==
+ *reinterpret_cast<const int32_t*>(offset2));
+ case OptionType::kInt64T:
+ {
+ int64_t v1, v2;
+ GetUnaligned(reinterpret_cast<const int64_t*>(offset1), &v1);
+ GetUnaligned(reinterpret_cast<const int64_t*>(offset2), &v2);
+ return (v1 == v2);
+ }
+ case OptionType::kVectorInt:
+ return (*reinterpret_cast<const std::vector<int>*>(offset1) ==
+ *reinterpret_cast<const std::vector<int>*>(offset2));
+ case OptionType::kUInt:
+ return (*reinterpret_cast<const unsigned int*>(offset1) ==
+ *reinterpret_cast<const unsigned int*>(offset2));
+ case OptionType::kUInt32T:
+ return (*reinterpret_cast<const uint32_t*>(offset1) ==
+ *reinterpret_cast<const uint32_t*>(offset2));
+ case OptionType::kUInt64T:
+ {
+ uint64_t v1, v2;
+ GetUnaligned(reinterpret_cast<const uint64_t*>(offset1), &v1);
+ GetUnaligned(reinterpret_cast<const uint64_t*>(offset2), &v2);
+ return (v1 == v2);
+ }
+ case OptionType::kSizeT:
+ {
+ size_t v1, v2;
+ GetUnaligned(reinterpret_cast<const size_t*>(offset1), &v1);
+ GetUnaligned(reinterpret_cast<const size_t*>(offset2), &v2);
+ return (v1 == v2);
+ }
+ case OptionType::kString:
+ return (*reinterpret_cast<const std::string*>(offset1) ==
+ *reinterpret_cast<const std::string*>(offset2));
+ case OptionType::kDouble:
+ return AreEqualDoubles(*reinterpret_cast<const double*>(offset1),
+ *reinterpret_cast<const double*>(offset2));
+ case OptionType::kCompactionStyle:
+ return (*reinterpret_cast<const CompactionStyle*>(offset1) ==
+ *reinterpret_cast<const CompactionStyle*>(offset2));
+ case OptionType::kCompactionPri:
+ return (*reinterpret_cast<const CompactionPri*>(offset1) ==
+ *reinterpret_cast<const CompactionPri*>(offset2));
+ case OptionType::kCompressionType:
+ return (*reinterpret_cast<const CompressionType*>(offset1) ==
+ *reinterpret_cast<const CompressionType*>(offset2));
+ case OptionType::kVectorCompressionType: {
+ const auto* vec1 =
+ reinterpret_cast<const std::vector<CompressionType>*>(offset1);
+ const auto* vec2 =
+ reinterpret_cast<const std::vector<CompressionType>*>(offset2);
+ return (*vec1 == *vec2);
+ }
+ case OptionType::kChecksumType:
+ return (*reinterpret_cast<const ChecksumType*>(offset1) ==
+ *reinterpret_cast<const ChecksumType*>(offset2));
+ case OptionType::kBlockBasedTableIndexType:
+ return (
+ *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
+ offset1) ==
+ *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(offset2));
+ case OptionType::kBlockBasedTableDataBlockIndexType:
+ return (
+ *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
+ offset1) ==
+ *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
+ offset2));
+ case OptionType::kBlockBasedTableIndexShorteningMode:
+ return (
+ *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
+ offset1) ==
+ *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
+ offset2));
+ case OptionType::kWALRecoveryMode:
+ return (*reinterpret_cast<const WALRecoveryMode*>(offset1) ==
+ *reinterpret_cast<const WALRecoveryMode*>(offset2));
+ case OptionType::kAccessHint:
+ return (*reinterpret_cast<const DBOptions::AccessHint*>(offset1) ==
+ *reinterpret_cast<const DBOptions::AccessHint*>(offset2));
+ case OptionType::kInfoLogLevel:
+ return (*reinterpret_cast<const InfoLogLevel*>(offset1) ==
+ *reinterpret_cast<const InfoLogLevel*>(offset2));
+ case OptionType::kCompactionOptionsFIFO: {
+ CompactionOptionsFIFO lhs =
+ *reinterpret_cast<const CompactionOptionsFIFO*>(offset1);
+ CompactionOptionsFIFO rhs =
+ *reinterpret_cast<const CompactionOptionsFIFO*>(offset2);
+ if (lhs.max_table_files_size == rhs.max_table_files_size &&
+ lhs.allow_compaction == rhs.allow_compaction) {
+ return true;
+ }
+ return false;
+ }
+ case OptionType::kCompactionOptionsUniversal: {
+ CompactionOptionsUniversal lhs =
+ *reinterpret_cast<const CompactionOptionsUniversal*>(offset1);
+ CompactionOptionsUniversal rhs =
+ *reinterpret_cast<const CompactionOptionsUniversal*>(offset2);
+ if (lhs.size_ratio == rhs.size_ratio &&
+ lhs.min_merge_width == rhs.min_merge_width &&
+ lhs.max_merge_width == rhs.max_merge_width &&
+ lhs.max_size_amplification_percent ==
+ rhs.max_size_amplification_percent &&
+ lhs.compression_size_percent == rhs.compression_size_percent &&
+ lhs.stop_style == rhs.stop_style &&
+ lhs.allow_trivial_move == rhs.allow_trivial_move) {
+ return true;
+ }
+ return false;
+ }
+ default:
+ if (type_info.verification == OptionVerificationType::kByName ||
+ type_info.verification ==
+ OptionVerificationType::kByNameAllowFromNull ||
+ type_info.verification == OptionVerificationType::kByNameAllowNull) {
+ std::string value1;
+ bool result =
+ SerializeSingleOptionHelper(offset1, type_info.type, &value1);
+ if (result == false) {
+ return false;
+ }
+ if (opt_map == nullptr) {
+ return true;
+ }
+ auto iter = opt_map->find(opt_name);
+ if (iter == opt_map->end()) {
+ return true;
+ } else {
+ if (type_info.verification ==
+ OptionVerificationType::kByNameAllowNull) {
+ if (iter->second == kNullptrString || value1 == kNullptrString) {
+ return true;
+ }
+ } else if (type_info.verification ==
+ OptionVerificationType::kByNameAllowFromNull) {
+ if (iter->second == kNullptrString) {
+ return true;
+ }
+ }
+ return (value1 == iter->second);
+ }
+ }
+ return false;
+ }
+}
+
+Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ const DBOptions& db_opt, const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs,
+ OptionsSanityCheckLevel sanity_check_level, bool ignore_unknown_options) {
+ // We infer option file readhead size from log readahead size.
+ // If it is not given, use 512KB.
+ size_t file_readahead_size = db_opt.log_readahead_size;
+ if (file_readahead_size == 0) {
+ const size_t kDefaultOptionFileReadAheadSize = 512 * 1024;
+ file_readahead_size = kDefaultOptionFileReadAheadSize;
+ }
+
+ RocksDBOptionsParser parser;
+ Status s =
+ parser.Parse(file_name, fs, ignore_unknown_options, file_readahead_size);
+ if (!s.ok()) {
+ return s;
+ }
+
+ // Verify DBOptions
+ s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(),
+ sanity_check_level);
+ if (!s.ok()) {
+ return s;
+ }
+
+ // Verify ColumnFamily Name
+ if (cf_names.size() != parser.cf_names()->size()) {
+ if (sanity_check_level >= kSanityLevelLooselyCompatible) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionParser Error] The persisted options does not have "
+ "the same number of column family names as the db instance.");
+ } else if (cf_opts.size() > parser.cf_opts()->size()) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error]",
+ "The persisted options file has less number of column family "
+ "names than that of the specified one.");
+ }
+ }
+ for (size_t i = 0; i < cf_names.size(); ++i) {
+ if (cf_names[i] != parser.cf_names()->at(i)) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionParser Error] The persisted options and the db"
+ "instance does not have the same name for column family ",
+ ToString(i));
+ }
+ }
+
+ // Verify Column Family Options
+ if (cf_opts.size() != parser.cf_opts()->size()) {
+ if (sanity_check_level >= kSanityLevelLooselyCompatible) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error]",
+ "The persisted options does not have the same number of "
+ "column families as the db instance.");
+ } else if (cf_opts.size() > parser.cf_opts()->size()) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error]",
+ "The persisted options file has less number of column families "
+ "than that of the specified number.");
+ }
+ }
+ for (size_t i = 0; i < cf_opts.size(); ++i) {
+ s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i),
+ &(parser.cf_opt_maps()->at(i)), sanity_check_level);
+ if (!s.ok()) {
+ return s;
+ }
+ s = VerifyTableFactory(cf_opts[i].table_factory.get(),
+ parser.cf_opts()->at(i).table_factory.get(),
+ sanity_check_level);
+ if (!s.ok()) {
+ return s;
+ }
+ }
+
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::VerifyDBOptions(
+ const DBOptions& base_opt, const DBOptions& persisted_opt,
+ const std::unordered_map<std::string, std::string>* /*opt_map*/,
+ OptionsSanityCheckLevel sanity_check_level) {
+ for (auto pair : db_options_type_info) {
+ if (pair.second.verification == OptionVerificationType::kDeprecated) {
+ // We skip checking deprecated variables as they might
+ // contain random values since they might not be initialized
+ continue;
+ }
+ if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
+ if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
+ reinterpret_cast<const char*>(&persisted_opt),
+ pair.second, pair.first, nullptr)) {
+ constexpr size_t kBufferSize = 2048;
+ char buffer[kBufferSize];
+ std::string base_value;
+ std::string persisted_value;
+ SerializeSingleOptionHelper(
+ reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
+ pair.second.type, &base_value);
+ SerializeSingleOptionHelper(
+ reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
+ pair.second.type, &persisted_value);
+ snprintf(buffer, sizeof(buffer),
+ "[RocksDBOptionsParser]: "
+ "failed the verification on DBOptions::%s --- "
+ "The specified one is %s while the persisted one is %s.\n",
+ pair.first.c_str(), base_value.c_str(),
+ persisted_value.c_str());
+ return Status::InvalidArgument(Slice(buffer, strlen(buffer)));
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::VerifyCFOptions(
+ const ColumnFamilyOptions& base_opt,
+ const ColumnFamilyOptions& persisted_opt,
+ const std::unordered_map<std::string, std::string>* persisted_opt_map,
+ OptionsSanityCheckLevel sanity_check_level) {
+ for (auto& pair : cf_options_type_info) {
+ if (pair.second.verification == OptionVerificationType::kDeprecated) {
+ // We skip checking deprecated variables as they might
+ // contain random values since they might not be initialized
+ continue;
+ }
+ if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
+ if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
+ reinterpret_cast<const char*>(&persisted_opt),
+ pair.second, pair.first, persisted_opt_map)) {
+ constexpr size_t kBufferSize = 2048;
+ char buffer[kBufferSize];
+ std::string base_value;
+ std::string persisted_value;
+ SerializeSingleOptionHelper(
+ reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
+ pair.second.type, &base_value);
+ SerializeSingleOptionHelper(
+ reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
+ pair.second.type, &persisted_value);
+ snprintf(buffer, sizeof(buffer),
+ "[RocksDBOptionsParser]: "
+ "failed the verification on ColumnFamilyOptions::%s --- "
+ "The specified one is %s while the persisted one is %s.\n",
+ pair.first.c_str(), base_value.c_str(),
+ persisted_value.c_str());
+ return Status::InvalidArgument(Slice(buffer, sizeof(buffer)));
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::VerifyTableFactory(
+ const TableFactory* base_tf, const TableFactory* file_tf,
+ OptionsSanityCheckLevel sanity_check_level) {
+ if (base_tf && file_tf) {
+ if (sanity_check_level > kSanityLevelNone &&
+ std::string(base_tf->Name()) != std::string(file_tf->Name())) {
+ return Status::Corruption(
+ "[RocksDBOptionsParser]: "
+ "failed the verification on TableFactory->Name()");
+ }
+ if (base_tf->Name() == BlockBasedTableFactory::kName) {
+ return VerifyBlockBasedTableFactory(
+ static_cast_with_check<const BlockBasedTableFactory,
+ const TableFactory>(base_tf),
+ static_cast_with_check<const BlockBasedTableFactory,
+ const TableFactory>(file_tf),
+ sanity_check_level);
+ }
+ // TODO(yhchiang): add checks for other table factory types
+ } else {
+ // TODO(yhchiang): further support sanity check here
+ }
+ return Status::OK();
+}
+} // namespace ROCKSDB_NAMESPACE
+
+#endif // !ROCKSDB_LITE
diff --git a/src/rocksdb/options/options_parser.h b/src/rocksdb/options/options_parser.h
new file mode 100644
index 000000000..a88b806f8
--- /dev/null
+++ b/src/rocksdb/options/options_parser.h
@@ -0,0 +1,147 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "options/options_sanity_check.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "table/block_based/block_based_table_factory.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+#ifndef ROCKSDB_LITE
+
+#define ROCKSDB_OPTION_FILE_MAJOR 1
+#define ROCKSDB_OPTION_FILE_MINOR 1
+
+enum OptionSection : char {
+ kOptionSectionVersion = 0,
+ kOptionSectionDBOptions,
+ kOptionSectionCFOptions,
+ kOptionSectionTableOptions,
+ kOptionSectionUnknown
+};
+
+static const std::string opt_section_titles[] = {
+ "Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"};
+
+Status PersistRocksDBOptions(const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs);
+
+extern bool AreEqualOptions(
+ const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
+ const std::string& opt_name,
+ const std::unordered_map<std::string, std::string>* opt_map);
+
+class RocksDBOptionsParser {
+ public:
+ explicit RocksDBOptionsParser();
+ ~RocksDBOptionsParser() {}
+ void Reset();
+
+ // `file_readahead_size` is used for readahead for the option file.
+ // If 0 is given, a default value will be used.
+ Status Parse(const std::string& file_name, FileSystem* fs,
+ bool ignore_unknown_options, size_t file_readahead_size);
+ static std::string TrimAndRemoveComment(const std::string& line,
+ const bool trim_only = false);
+
+ const DBOptions* db_opt() const { return &db_opt_; }
+ const std::unordered_map<std::string, std::string>* db_opt_map() const {
+ return &db_opt_map_;
+ }
+ const std::vector<ColumnFamilyOptions>* cf_opts() const { return &cf_opts_; }
+ const std::vector<std::string>* cf_names() const { return &cf_names_; }
+ const std::vector<std::unordered_map<std::string, std::string>>* cf_opt_maps()
+ const {
+ return &cf_opt_maps_;
+ }
+
+ const ColumnFamilyOptions* GetCFOptions(const std::string& name) {
+ return GetCFOptionsImpl(name);
+ }
+ size_t NumColumnFamilies() { return cf_opts_.size(); }
+
+ static Status VerifyRocksDBOptionsFromFile(
+ const DBOptions& db_opt, const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs,
+ OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch,
+ bool ignore_unknown_options = false);
+
+ static Status VerifyDBOptions(
+ const DBOptions& base_opt, const DBOptions& new_opt,
+ const std::unordered_map<std::string, std::string>* new_opt_map = nullptr,
+ OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
+
+ static Status VerifyCFOptions(
+ const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
+ const std::unordered_map<std::string, std::string>* new_opt_map = nullptr,
+ OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
+
+ static Status VerifyTableFactory(
+ const TableFactory* base_tf, const TableFactory* file_tf,
+ OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
+
+ static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
+
+ protected:
+ bool IsSection(const std::string& line);
+ Status ParseSection(OptionSection* section, std::string* title,
+ std::string* argument, const std::string& line,
+ const int line_num);
+
+ Status CheckSection(const OptionSection section,
+ const std::string& section_arg, const int line_num);
+
+ Status ParseStatement(std::string* name, std::string* value,
+ const std::string& line, const int line_num);
+
+ Status EndSection(const OptionSection section, const std::string& title,
+ const std::string& section_arg,
+ const std::unordered_map<std::string, std::string>& opt_map,
+ bool ignore_unknown_options);
+
+ Status ValidityCheck();
+
+ Status InvalidArgument(const int line_num, const std::string& message);
+
+ Status ParseVersionNumber(const std::string& ver_name,
+ const std::string& ver_string, const int max_count,
+ int* version);
+
+ ColumnFamilyOptions* GetCFOptionsImpl(const std::string& name) {
+ assert(cf_names_.size() == cf_opts_.size());
+ for (size_t i = 0; i < cf_names_.size(); ++i) {
+ if (cf_names_[i] == name) {
+ return &cf_opts_[i];
+ }
+ }
+ return nullptr;
+ }
+
+ private:
+ DBOptions db_opt_;
+ std::unordered_map<std::string, std::string> db_opt_map_;
+ std::vector<std::string> cf_names_;
+ std::vector<ColumnFamilyOptions> cf_opts_;
+ std::vector<std::unordered_map<std::string, std::string>> cf_opt_maps_;
+ bool has_version_section_;
+ bool has_db_options_;
+ bool has_default_cf_options_;
+ int db_version[3];
+ int opt_file_version[3];
+};
+
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_sanity_check.cc b/src/rocksdb/options/options_sanity_check.cc
new file mode 100644
index 000000000..47cff31cf
--- /dev/null
+++ b/src/rocksdb/options/options_sanity_check.cc
@@ -0,0 +1,38 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include "options/options_sanity_check.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+namespace {
+OptionsSanityCheckLevel SanityCheckLevelHelper(
+ const std::unordered_map<std::string, OptionsSanityCheckLevel>& smap,
+ const std::string& name) {
+ auto iter = smap.find(name);
+ return iter != smap.end() ? iter->second : kSanityLevelExactMatch;
+}
+}
+
+OptionsSanityCheckLevel DBOptionSanityCheckLevel(
+ const std::string& option_name) {
+ return SanityCheckLevelHelper(sanity_level_db_options, option_name);
+}
+
+OptionsSanityCheckLevel CFOptionSanityCheckLevel(
+ const std::string& option_name) {
+ return SanityCheckLevelHelper(sanity_level_cf_options, option_name);
+}
+
+OptionsSanityCheckLevel BBTOptionSanityCheckLevel(
+ const std::string& option_name) {
+ return SanityCheckLevelHelper(sanity_level_bbt_options, option_name);
+}
+
+} // namespace ROCKSDB_NAMESPACE
+
+#endif // !ROCKSDB_LITE
diff --git a/src/rocksdb/options/options_sanity_check.h b/src/rocksdb/options/options_sanity_check.h
new file mode 100644
index 000000000..ab8ec6d4d
--- /dev/null
+++ b/src/rocksdb/options/options_sanity_check.h
@@ -0,0 +1,50 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+#include "rocksdb/rocksdb_namespace.h"
+
+#ifndef ROCKSDB_LITE
+namespace ROCKSDB_NAMESPACE {
+// This enum defines the RocksDB options sanity level.
+enum OptionsSanityCheckLevel : unsigned char {
+ // Performs no sanity check at all.
+ kSanityLevelNone = 0x00,
+ // Performs minimum check to ensure the RocksDB instance can be
+ // opened without corrupting / mis-interpreting the data.
+ kSanityLevelLooselyCompatible = 0x01,
+ // Perform exact match sanity check.
+ kSanityLevelExactMatch = 0xFF,
+};
+
+// The sanity check level for DB options
+static const std::unordered_map<std::string, OptionsSanityCheckLevel>
+ sanity_level_db_options {};
+
+// The sanity check level for column-family options
+static const std::unordered_map<std::string, OptionsSanityCheckLevel>
+ sanity_level_cf_options = {
+ {"comparator", kSanityLevelLooselyCompatible},
+ {"table_factory", kSanityLevelLooselyCompatible},
+ {"merge_operator", kSanityLevelLooselyCompatible}};
+
+// The sanity check level for block-based table options
+static const std::unordered_map<std::string, OptionsSanityCheckLevel>
+ sanity_level_bbt_options {};
+
+OptionsSanityCheckLevel DBOptionSanityCheckLevel(
+ const std::string& options_name);
+OptionsSanityCheckLevel CFOptionSanityCheckLevel(
+ const std::string& options_name);
+OptionsSanityCheckLevel BBTOptionSanityCheckLevel(
+ const std::string& options_name);
+
+} // namespace ROCKSDB_NAMESPACE
+
+#endif // !ROCKSDB_LITE
diff --git a/src/rocksdb/options/options_settable_test.cc b/src/rocksdb/options/options_settable_test.cc
new file mode 100644
index 000000000..c5fee7b3e
--- /dev/null
+++ b/src/rocksdb/options/options_settable_test.cc
@@ -0,0 +1,492 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <cstring>
+
+#include "options/options_helper.h"
+#include "rocksdb/convenience.h"
+#include "test_util/testharness.h"
+
+#ifndef GFLAGS
+bool FLAGS_enable_print = false;
+#else
+#include "util/gflags_compat.h"
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+DEFINE_bool(enable_print, false, "Print options generated to console.");
+#endif // GFLAGS
+
+namespace ROCKSDB_NAMESPACE {
+
+// Verify options are settable from options strings.
+// We take the approach that depends on compiler behavior that copy constructor
+// won't touch implicit padding bytes, so that the test is fragile.
+// As a result, we only run the tests to verify new fields in options are
+// settable through string on limited platforms as it depends on behavior of
+// compilers.
+#ifndef ROCKSDB_LITE
+#if defined OS_LINUX || defined OS_WIN
+#ifndef __clang__
+
+class OptionsSettableTest : public testing::Test {
+ public:
+ OptionsSettableTest() {}
+};
+
+const char kSpecialChar = 'z';
+typedef std::vector<std::pair<size_t, size_t>> OffsetGap;
+
+void FillWithSpecialChar(char* start_ptr, size_t total_size,
+ const OffsetGap& blacklist) {
+ size_t offset = 0;
+ for (auto& pair : blacklist) {
+ std::memset(start_ptr + offset, kSpecialChar, pair.first - offset);
+ offset = pair.first + pair.second;
+ }
+ std::memset(start_ptr + offset, kSpecialChar, total_size - offset);
+}
+
+int NumUnsetBytes(char* start_ptr, size_t total_size,
+ const OffsetGap& blacklist) {
+ int total_unset_bytes_base = 0;
+ size_t offset = 0;
+ for (auto& pair : blacklist) {
+ for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) {
+ if (*ptr == kSpecialChar) {
+ total_unset_bytes_base++;
+ }
+ }
+ offset = pair.first + pair.second;
+ }
+ for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) {
+ if (*ptr == kSpecialChar) {
+ total_unset_bytes_base++;
+ }
+ }
+ return total_unset_bytes_base;
+}
+
+// If the test fails, likely a new option is added to BlockBasedTableOptions
+// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
+// test is not updated accordingly.
+// After adding an option, we need to make sure it is settable by
+// GetBlockBasedTableOptionsFromString() and add the option to the input string
+// passed to the GetBlockBasedTableOptionsFromString() in this test.
+// If it is a complicated type, you also need to add the field to
+// kBbtoBlacklist, and maybe add customized verification for it.
+TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
+ // Items in the form of <offset, size>. Need to be in ascending order
+ // and not overlapping. Need to updated if new pointer-option is added.
+ const OffsetGap kBbtoBlacklist = {
+ {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
+ sizeof(std::shared_ptr<FlushBlockPolicyFactory>)},
+ {offsetof(struct BlockBasedTableOptions, block_cache),
+ sizeof(std::shared_ptr<Cache>)},
+ {offsetof(struct BlockBasedTableOptions, persistent_cache),
+ sizeof(std::shared_ptr<PersistentCache>)},
+ {offsetof(struct BlockBasedTableOptions, block_cache_compressed),
+ sizeof(std::shared_ptr<Cache>)},
+ {offsetof(struct BlockBasedTableOptions, filter_policy),
+ sizeof(std::shared_ptr<const FilterPolicy>)},
+ };
+
+ // In this test, we catch a new option of BlockBasedTableOptions that is not
+ // settable through GetBlockBasedTableOptionsFromString().
+ // We count padding bytes of the option struct, and assert it to be the same
+ // as unset bytes of an option struct initialized by
+ // GetBlockBasedTableOptionsFromString().
+
+ char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
+
+ // Count padding bytes by setting all bytes in the memory to a special char,
+ // copy a well constructed struct to this memory and see how many special
+ // bytes left.
+ BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions();
+ FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
+ // It based on the behavior of compiler that padding bytes are not changed
+ // when copying the struct. It's prone to failure when compiler behavior
+ // changes. We verify there is unset bytes to detect the case.
+ *bbto = BlockBasedTableOptions();
+ int unset_bytes_base =
+ NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
+ ASSERT_GT(unset_bytes_base, 0);
+ bbto->~BlockBasedTableOptions();
+
+ // Construct the base option passed into
+ // GetBlockBasedTableOptionsFromString().
+ bbto = new (bbto_ptr) BlockBasedTableOptions();
+ FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
+ // This option is not setable:
+ bbto->use_delta_encoding = true;
+
+ char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
+ BlockBasedTableOptions* new_bbto =
+ new (new_bbto_ptr) BlockBasedTableOptions();
+ FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions),
+ kBbtoBlacklist);
+
+ // Need to update the option string if a new option is added.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ *bbto,
+ "cache_index_and_filter_blocks=1;"
+ "cache_index_and_filter_blocks_with_high_priority=true;"
+ "pin_l0_filter_and_index_blocks_in_cache=1;"
+ "pin_top_level_index_and_filter=1;"
+ "index_type=kHashSearch;"
+ "data_block_index_type=kDataBlockBinaryAndHash;"
+ "index_shortening=kNoShortening;"
+ "data_block_hash_table_util_ratio=0.75;"
+ "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
+ "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
+ "block_size_deviation=8;block_restart_interval=4; "
+ "metadata_block_size=1024;"
+ "partition_filters=false;"
+ "index_block_restart_interval=4;"
+ "filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
+ "format_version=1;"
+ "hash_index_allow_collision=false;"
+ "verify_compression=true;read_amp_bytes_per_bit=0;"
+ "enable_index_compression=false;"
+ "block_align=true",
+ new_bbto));
+
+ ASSERT_EQ(unset_bytes_base,
+ NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions),
+ kBbtoBlacklist));
+
+ ASSERT_TRUE(new_bbto->block_cache.get() != nullptr);
+ ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr);
+ ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr);
+
+ bbto->~BlockBasedTableOptions();
+ new_bbto->~BlockBasedTableOptions();
+
+ delete[] bbto_ptr;
+ delete[] new_bbto_ptr;
+}
+
+// If the test fails, likely a new option is added to DBOptions
+// but it cannot be set through GetDBOptionsFromString(), or the test is not
+// updated accordingly.
+// After adding an option, we need to make sure it is settable by
+// GetDBOptionsFromString() and add the option to the input string passed to
+// DBOptionsFromString()in this test.
+// If it is a complicated type, you also need to add the field to
+// kDBOptionsBlacklist, and maybe add customized verification for it.
+TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
+ const OffsetGap kDBOptionsBlacklist = {
+ {offsetof(struct DBOptions, env), sizeof(Env*)},
+ {offsetof(struct DBOptions, file_system),
+ sizeof(std::shared_ptr<FileSystem>)},
+ {offsetof(struct DBOptions, rate_limiter),
+ sizeof(std::shared_ptr<RateLimiter>)},
+ {offsetof(struct DBOptions, sst_file_manager),
+ sizeof(std::shared_ptr<SstFileManager>)},
+ {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)},
+ {offsetof(struct DBOptions, statistics),
+ sizeof(std::shared_ptr<Statistics>)},
+ {offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)},
+ {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)},
+ {offsetof(struct DBOptions, wal_dir), sizeof(std::string)},
+ {offsetof(struct DBOptions, write_buffer_manager),
+ sizeof(std::shared_ptr<WriteBufferManager>)},
+ {offsetof(struct DBOptions, listeners),
+ sizeof(std::vector<std::shared_ptr<EventListener>>)},
+ {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)},
+ {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)},
+ {offsetof(struct DBOptions, sst_file_checksum_func),
+ sizeof(std::shared_ptr<FileChecksumFunc>)},
+ };
+
+ char* options_ptr = new char[sizeof(DBOptions)];
+
+ // Count padding bytes by setting all bytes in the memory to a special char,
+ // copy a well constructed struct to this memory and see how many special
+ // bytes left.
+ DBOptions* options = new (options_ptr) DBOptions();
+ FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
+ // It based on the behavior of compiler that padding bytes are not changed
+ // when copying the struct. It's prone to failure when compiler behavior
+ // changes. We verify there is unset bytes to detect the case.
+ *options = DBOptions();
+ int unset_bytes_base =
+ NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
+ ASSERT_GT(unset_bytes_base, 0);
+ options->~DBOptions();
+
+ options = new (options_ptr) DBOptions();
+ FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
+
+ char* new_options_ptr = new char[sizeof(DBOptions)];
+ DBOptions* new_options = new (new_options_ptr) DBOptions();
+ FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
+
+ // Need to update the option string if a new option is added.
+ ASSERT_OK(
+ GetDBOptionsFromString(*options,
+ "wal_bytes_per_sync=4295048118;"
+ "delete_obsolete_files_period_micros=4294967758;"
+ "WAL_ttl_seconds=4295008036;"
+ "WAL_size_limit_MB=4295036161;"
+ "max_write_batch_group_size_bytes=1048576;"
+ "wal_dir=path/to/wal_dir;"
+ "db_write_buffer_size=2587;"
+ "max_subcompactions=64330;"
+ "table_cache_numshardbits=28;"
+ "max_open_files=72;"
+ "max_file_opening_threads=35;"
+ "max_background_jobs=8;"
+ "base_background_compactions=3;"
+ "max_background_compactions=33;"
+ "use_fsync=true;"
+ "use_adaptive_mutex=false;"
+ "max_total_wal_size=4295005604;"
+ "compaction_readahead_size=0;"
+ "new_table_reader_for_compaction_inputs=false;"
+ "keep_log_file_num=4890;"
+ "skip_stats_update_on_db_open=false;"
+ "skip_checking_sst_file_sizes_on_db_open=false;"
+ "max_manifest_file_size=4295009941;"
+ "db_log_dir=path/to/db_log_dir;"
+ "skip_log_error_on_recovery=true;"
+ "writable_file_max_buffer_size=1048576;"
+ "paranoid_checks=true;"
+ "is_fd_close_on_exec=false;"
+ "bytes_per_sync=4295013613;"
+ "strict_bytes_per_sync=true;"
+ "enable_thread_tracking=false;"
+ "recycle_log_file_num=0;"
+ "create_missing_column_families=true;"
+ "log_file_time_to_roll=3097;"
+ "max_background_flushes=35;"
+ "create_if_missing=false;"
+ "error_if_exists=true;"
+ "delayed_write_rate=4294976214;"
+ "manifest_preallocation_size=1222;"
+ "allow_mmap_writes=false;"
+ "stats_dump_period_sec=70127;"
+ "stats_persist_period_sec=54321;"
+ "persist_stats_to_disk=true;"
+ "stats_history_buffer_size=14159;"
+ "allow_fallocate=true;"
+ "allow_mmap_reads=false;"
+ "use_direct_reads=false;"
+ "use_direct_io_for_flush_and_compaction=false;"
+ "max_log_file_size=4607;"
+ "random_access_max_buffer_size=1048576;"
+ "advise_random_on_open=true;"
+ "fail_if_options_file_error=false;"
+ "enable_pipelined_write=false;"
+ "unordered_write=false;"
+ "allow_concurrent_memtable_write=true;"
+ "wal_recovery_mode=kPointInTimeRecovery;"
+ "enable_write_thread_adaptive_yield=true;"
+ "write_thread_slow_yield_usec=5;"
+ "write_thread_max_yield_usec=1000;"
+ "access_hint_on_compaction_start=NONE;"
+ "info_log_level=DEBUG_LEVEL;"
+ "dump_malloc_stats=false;"
+ "allow_2pc=false;"
+ "avoid_flush_during_recovery=false;"
+ "avoid_flush_during_shutdown=false;"
+ "allow_ingest_behind=false;"
+ "preserve_deletes=false;"
+ "concurrent_prepare=false;"
+ "two_write_queues=false;"
+ "manual_wal_flush=false;"
+ "seq_per_batch=false;"
+ "atomic_flush=false;"
+ "avoid_unnecessary_blocking_io=false;"
+ "log_readahead_size=0;"
+ "write_dbid_to_manifest=false",
+ new_options));
+
+ ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
+ kDBOptionsBlacklist));
+
+ options->~DBOptions();
+ new_options->~DBOptions();
+
+ delete[] options_ptr;
+ delete[] new_options_ptr;
+}
+
+template <typename T1, typename T2>
+inline int offset_of(T1 T2::*member) {
+ static T2 obj;
+ return int(size_t(&(obj.*member)) - size_t(&obj));
+}
+
+// If the test fails, likely a new option is added to ColumnFamilyOptions
+// but it cannot be set through GetColumnFamilyOptionsFromString(), or the
+// test is not updated accordingly.
+// After adding an option, we need to make sure it is settable by
+// GetColumnFamilyOptionsFromString() and add the option to the input
+// string passed to GetColumnFamilyOptionsFromString()in this test.
+// If it is a complicated type, you also need to add the field to
+// kColumnFamilyOptionsBlacklist, and maybe add customized verification
+// for it.
+TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
+ // options in the blacklist need to appear in the same order as in
+ // ColumnFamilyOptions.
+ const OffsetGap kColumnFamilyOptionsBlacklist = {
+ {offset_of(&ColumnFamilyOptions::inplace_callback),
+ sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))},
+ {offset_of(
+ &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor),
+ sizeof(std::shared_ptr<const SliceTransform>)},
+ {offset_of(&ColumnFamilyOptions::compression_per_level),
+ sizeof(std::vector<CompressionType>)},
+ {offset_of(
+ &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional),
+ sizeof(std::vector<int>)},
+ {offset_of(&ColumnFamilyOptions::memtable_factory),
+ sizeof(std::shared_ptr<MemTableRepFactory>)},
+ {offset_of(&ColumnFamilyOptions::table_properties_collector_factories),
+ sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)},
+ {offset_of(&ColumnFamilyOptions::comparator), sizeof(Comparator*)},
+ {offset_of(&ColumnFamilyOptions::merge_operator),
+ sizeof(std::shared_ptr<MergeOperator>)},
+ {offset_of(&ColumnFamilyOptions::compaction_filter),
+ sizeof(const CompactionFilter*)},
+ {offset_of(&ColumnFamilyOptions::compaction_filter_factory),
+ sizeof(std::shared_ptr<CompactionFilterFactory>)},
+ {offset_of(&ColumnFamilyOptions::prefix_extractor),
+ sizeof(std::shared_ptr<const SliceTransform>)},
+ {offset_of(&ColumnFamilyOptions::snap_refresh_nanos), sizeof(uint64_t)},
+ {offset_of(&ColumnFamilyOptions::table_factory),
+ sizeof(std::shared_ptr<TableFactory>)},
+ {offset_of(&ColumnFamilyOptions::cf_paths), sizeof(std::vector<DbPath>)},
+ {offset_of(&ColumnFamilyOptions::compaction_thread_limiter),
+ sizeof(std::shared_ptr<ConcurrentTaskLimiter>)},
+ };
+
+ char* options_ptr = new char[sizeof(ColumnFamilyOptions)];
+
+ // Count padding bytes by setting all bytes in the memory to a special char,
+ // copy a well constructed struct to this memory and see how many special
+ // bytes left.
+ ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions();
+ FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsBlacklist);
+ // It based on the behavior of compiler that padding bytes are not changed
+ // when copying the struct. It's prone to failure when compiler behavior
+ // changes. We verify there is unset bytes to detect the case.
+ *options = ColumnFamilyOptions();
+
+ // Deprecatd option which is not initialized. Need to set it to avoid
+ // Valgrind error
+ options->max_mem_compaction_level = 0;
+
+ int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsBlacklist);
+ ASSERT_GT(unset_bytes_base, 0);
+ options->~ColumnFamilyOptions();
+
+ options = new (options_ptr) ColumnFamilyOptions();
+ FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsBlacklist);
+
+ // Following options are not settable through
+ // GetColumnFamilyOptionsFromString():
+ options->rate_limit_delay_max_milliseconds = 33;
+ options->compaction_options_universal = CompactionOptionsUniversal();
+ options->compression_opts = CompressionOptions();
+ options->bottommost_compression_opts = CompressionOptions();
+ options->hard_rate_limit = 0;
+ options->soft_rate_limit = 0;
+ options->purge_redundant_kvs_while_flush = false;
+ options->max_mem_compaction_level = 0;
+ options->compaction_filter = nullptr;
+
+ char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
+ ColumnFamilyOptions* new_options =
+ new (new_options_ptr) ColumnFamilyOptions();
+ FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsBlacklist);
+
+ // Need to update the option string if a new option is added.
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ *options,
+ "compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
+ "table_factory=PlainTable;"
+ "prefix_extractor=rocksdb.CappedPrefix.13;"
+ "comparator=leveldb.BytewiseComparator;"
+ "compression_per_level=kBZip2Compression:kBZip2Compression:"
+ "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
+ "kSnappyCompression;"
+ "max_bytes_for_level_base=986;"
+ "bloom_locality=8016;"
+ "target_file_size_base=4294976376;"
+ "memtable_huge_page_size=2557;"
+ "max_successive_merges=5497;"
+ "max_sequential_skip_in_iterations=4294971408;"
+ "arena_block_size=1893;"
+ "target_file_size_multiplier=35;"
+ "min_write_buffer_number_to_merge=9;"
+ "max_write_buffer_number=84;"
+ "write_buffer_size=1653;"
+ "max_compaction_bytes=64;"
+ "max_bytes_for_level_multiplier=60;"
+ "memtable_factory=SkipListFactory;"
+ "compression=kNoCompression;"
+ "bottommost_compression=kDisableCompressionOption;"
+ "level0_stop_writes_trigger=33;"
+ "num_levels=99;"
+ "level0_slowdown_writes_trigger=22;"
+ "level0_file_num_compaction_trigger=14;"
+ "compaction_filter=urxcqstuwnCompactionFilter;"
+ "soft_rate_limit=530.615385;"
+ "soft_pending_compaction_bytes_limit=0;"
+ "max_write_buffer_number_to_maintain=84;"
+ "max_write_buffer_size_to_maintain=2147483648;"
+ "merge_operator=aabcxehazrMergeOperator;"
+ "memtable_prefix_bloom_size_ratio=0.4642;"
+ "memtable_whole_key_filtering=true;"
+ "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"
+ "paranoid_file_checks=true;"
+ "force_consistency_checks=true;"
+ "inplace_update_num_locks=7429;"
+ "optimize_filters_for_hits=false;"
+ "level_compaction_dynamic_level_bytes=false;"
+ "inplace_update_support=false;"
+ "compaction_style=kCompactionStyleFIFO;"
+ "compaction_pri=kMinOverlappingRatio;"
+ "hard_pending_compaction_bytes_limit=0;"
+ "disable_auto_compactions=false;"
+ "report_bg_io_stats=true;"
+ "ttl=60;"
+ "periodic_compaction_seconds=3600;"
+ "sample_for_compression=0;"
+ "compaction_options_fifo={max_table_files_size=3;allow_"
+ "compaction=false;};",
+ new_options));
+
+ ASSERT_EQ(unset_bytes_base,
+ NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsBlacklist));
+
+ options->~ColumnFamilyOptions();
+ new_options->~ColumnFamilyOptions();
+
+ delete[] options_ptr;
+ delete[] new_options_ptr;
+}
+#endif // !__clang__
+#endif // OS_LINUX || OS_WIN
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+#ifdef GFLAGS
+ ParseCommandLineFlags(&argc, &argv, true);
+#endif // GFLAGS
+ return RUN_ALL_TESTS();
+}
diff --git a/src/rocksdb/options/options_test.cc b/src/rocksdb/options/options_test.cc
new file mode 100644
index 000000000..9ff99532c
--- /dev/null
+++ b/src/rocksdb/options/options_test.cc
@@ -0,0 +1,2004 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <cctype>
+#include <cinttypes>
+#include <cstring>
+#include <unordered_map>
+
+#include "cache/lru_cache.h"
+#include "cache/sharded_cache.h"
+#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "options/options_sanity_check.h"
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/utilities/leveldb_options.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "table/block_based/filter_policy_internal.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+#include "util/random.h"
+#include "util/stderr_logger.h"
+#include "util/string_util.h"
+#include "utilities/merge_operators/bytesxor.h"
+
+#ifndef GFLAGS
+bool FLAGS_enable_print = false;
+#else
+#include "util/gflags_compat.h"
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+DEFINE_bool(enable_print, false, "Print options generated to console.");
+#endif // GFLAGS
+
+namespace ROCKSDB_NAMESPACE {
+
+class OptionsTest : public testing::Test {};
+
+#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
+TEST_F(OptionsTest, GetOptionsFromMapTest) {
+ std::unordered_map<std::string, std::string> cf_options_map = {
+ {"write_buffer_size", "1"},
+ {"max_write_buffer_number", "2"},
+ {"min_write_buffer_number_to_merge", "3"},
+ {"max_write_buffer_number_to_maintain", "99"},
+ {"max_write_buffer_size_to_maintain", "-99999"},
+ {"compression", "kSnappyCompression"},
+ {"compression_per_level",
+ "kNoCompression:"
+ "kSnappyCompression:"
+ "kZlibCompression:"
+ "kBZip2Compression:"
+ "kLZ4Compression:"
+ "kLZ4HCCompression:"
+ "kXpressCompression:"
+ "kZSTD:"
+ "kZSTDNotFinalCompression"},
+ {"bottommost_compression", "kLZ4Compression"},
+ {"bottommost_compression_opts", "5:6:7:8:9:true"},
+ {"compression_opts", "4:5:6:7:8:true"},
+ {"num_levels", "8"},
+ {"level0_file_num_compaction_trigger", "8"},
+ {"level0_slowdown_writes_trigger", "9"},
+ {"level0_stop_writes_trigger", "10"},
+ {"target_file_size_base", "12"},
+ {"target_file_size_multiplier", "13"},
+ {"max_bytes_for_level_base", "14"},
+ {"level_compaction_dynamic_level_bytes", "true"},
+ {"max_bytes_for_level_multiplier", "15.0"},
+ {"max_bytes_for_level_multiplier_additional", "16:17:18"},
+ {"max_compaction_bytes", "21"},
+ {"soft_rate_limit", "1.1"},
+ {"hard_rate_limit", "2.1"},
+ {"hard_pending_compaction_bytes_limit", "211"},
+ {"arena_block_size", "22"},
+ {"disable_auto_compactions", "true"},
+ {"compaction_style", "kCompactionStyleLevel"},
+ {"compaction_pri", "kOldestSmallestSeqFirst"},
+ {"verify_checksums_in_compaction", "false"},
+ {"compaction_options_fifo", "23"},
+ {"max_sequential_skip_in_iterations", "24"},
+ {"inplace_update_support", "true"},
+ {"report_bg_io_stats", "true"},
+ {"compaction_measure_io_stats", "false"},
+ {"inplace_update_num_locks", "25"},
+ {"memtable_prefix_bloom_size_ratio", "0.26"},
+ {"memtable_whole_key_filtering", "true"},
+ {"memtable_huge_page_size", "28"},
+ {"bloom_locality", "29"},
+ {"max_successive_merges", "30"},
+ {"min_partial_merge_operands", "31"},
+ {"prefix_extractor", "fixed:31"},
+ {"optimize_filters_for_hits", "true"},
+ };
+
+ std::unordered_map<std::string, std::string> db_options_map = {
+ {"create_if_missing", "false"},
+ {"create_missing_column_families", "true"},
+ {"error_if_exists", "false"},
+ {"paranoid_checks", "true"},
+ {"max_open_files", "32"},
+ {"max_total_wal_size", "33"},
+ {"use_fsync", "true"},
+ {"db_log_dir", "/db_log_dir"},
+ {"wal_dir", "/wal_dir"},
+ {"delete_obsolete_files_period_micros", "34"},
+ {"max_background_compactions", "35"},
+ {"max_background_flushes", "36"},
+ {"max_log_file_size", "37"},
+ {"log_file_time_to_roll", "38"},
+ {"keep_log_file_num", "39"},
+ {"recycle_log_file_num", "5"},
+ {"max_manifest_file_size", "40"},
+ {"table_cache_numshardbits", "41"},
+ {"WAL_ttl_seconds", "43"},
+ {"WAL_size_limit_MB", "44"},
+ {"manifest_preallocation_size", "45"},
+ {"allow_mmap_reads", "true"},
+ {"allow_mmap_writes", "false"},
+ {"use_direct_reads", "false"},
+ {"use_direct_io_for_flush_and_compaction", "false"},
+ {"is_fd_close_on_exec", "true"},
+ {"skip_log_error_on_recovery", "false"},
+ {"stats_dump_period_sec", "46"},
+ {"stats_persist_period_sec", "57"},
+ {"persist_stats_to_disk", "false"},
+ {"stats_history_buffer_size", "69"},
+ {"advise_random_on_open", "true"},
+ {"use_adaptive_mutex", "false"},
+ {"new_table_reader_for_compaction_inputs", "true"},
+ {"compaction_readahead_size", "100"},
+ {"random_access_max_buffer_size", "3145728"},
+ {"writable_file_max_buffer_size", "314159"},
+ {"bytes_per_sync", "47"},
+ {"wal_bytes_per_sync", "48"},
+ {"strict_bytes_per_sync", "true"},
+ };
+
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
+ ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999);
+ ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
+ ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
+ ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
+ ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
+ ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.num_levels, 8);
+ ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8);
+ ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9);
+ ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10);
+ ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12));
+ ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
+ ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18);
+ ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21);
+ ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
+ ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
+ ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
+ ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
+ ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
+ static_cast<uint64_t>(23));
+ ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
+ static_cast<uint64_t>(24));
+ ASSERT_EQ(new_cf_opt.inplace_update_support, true);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
+ ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
+ ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true);
+ ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
+ ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
+ ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
+ ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
+ ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),
+ "rocksdb.FixedPrefix.31");
+
+ cf_options_map["write_buffer_size"] = "hello";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ cf_options_map["write_buffer_size"] = "1";
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+
+ cf_options_map["unknown_option"] = "1";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
+ &new_cf_opt,
+ false, /* input_strings_escaped */
+ true /* ignore_unknown_options */));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
+ kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/));
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
+ kSanityLevelExactMatch /* default for VerifyCFOptions */));
+
+ DBOptions base_db_opt;
+ DBOptions new_db_opt;
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.create_missing_column_families, true);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.paranoid_checks, true);
+ ASSERT_EQ(new_db_opt.max_open_files, 32);
+ ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
+ ASSERT_EQ(new_db_opt.use_fsync, true);
+ ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir");
+ ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir");
+ ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros,
+ static_cast<uint64_t>(34));
+ ASSERT_EQ(new_db_opt.max_background_compactions, 35);
+ ASSERT_EQ(new_db_opt.max_background_flushes, 36);
+ ASSERT_EQ(new_db_opt.max_log_file_size, 37U);
+ ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U);
+ ASSERT_EQ(new_db_opt.keep_log_file_num, 39U);
+ ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U);
+ ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40));
+ ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41);
+ ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43));
+ ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44));
+ ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U);
+ ASSERT_EQ(new_db_opt.allow_mmap_reads, true);
+ ASSERT_EQ(new_db_opt.allow_mmap_writes, false);
+ ASSERT_EQ(new_db_opt.use_direct_reads, false);
+ ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false);
+ ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true);
+ ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false);
+ ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U);
+ ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U);
+ ASSERT_EQ(new_db_opt.persist_stats_to_disk, false);
+ ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U);
+ ASSERT_EQ(new_db_opt.advise_random_on_open, true);
+ ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
+ ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true);
+ ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
+ ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
+ ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
+ ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
+ ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
+ ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true);
+
+ db_options_map["max_open_files"] = "hello";
+ ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
+ base_db_opt, new_db_opt, nullptr, /* new_opt_map */
+ kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
+
+ // unknow options should fail parsing without ignore_unknown_options = true
+ db_options_map["unknown_db_option"] = "1";
+ ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
+
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt,
+ false, /* input_strings_escaped */
+ true /* ignore_unknown_options */));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
+ base_db_opt, new_db_opt, nullptr, /* new_opt_map */
+ kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
+ ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(
+ base_db_opt, new_db_opt, nullptr, /* new_opt_mat */
+ kSanityLevelExactMatch /* default for VerifyDBOptions */));
+}
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in
+ // ROCKSDB_LITE
+TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ base_cf_opt.table_factory.reset();
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=5", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
+ ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=6;", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ " write_buffer_size = 7 ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ " write_buffer_size = 8 ; ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=11; max_write_buffer_number = 12 ;",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
+ // Wrong name "max_write_buffer_number_"
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number_=14;",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ // Comparator from object registry
+ std::string kCompName = "reverse_comp";
+ ObjectLibrary::Default()->Register<const Comparator>(
+ kCompName,
+ [](const std::string& /*name*/,
+ std::unique_ptr<const Comparator>* /*guard*/,
+ std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator());
+
+ // MergeOperator from object registry
+ std::unique_ptr<BytesXOROperator> bxo(new BytesXOROperator());
+ std::string kMoName = bxo->Name();
+ ObjectLibrary::Default()->Register<MergeOperator>(
+ kMoName,
+ [](const std::string& /*name*/, std::unique_ptr<MergeOperator>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new BytesXOROperator());
+ return guard->get();
+ });
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "merge_operator=" + kMoName + ";", &new_cf_opt));
+ ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name()));
+
+ // Wrong key/value pair
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ // Error Paring value
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ // Missing option name
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13; =100;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ const uint64_t kilo = 1024UL;
+ const uint64_t mega = 1024 * kilo;
+ const uint64_t giga = 1024 * mega;
+ const uint64_t tera = 1024 * giga;
+
+ // Units (k)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo);
+ // Units (m)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "max_write_buffer_number=16m;inplace_update_num_locks=17M",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega);
+ // Units (g)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "write_buffer_size=18g;prefix_extractor=capped:8;"
+ "arena_block_size=19G",
+ &new_cf_opt));
+
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
+ std::string prefix_name(new_cf_opt.prefix_extractor->Name());
+ ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
+
+ // Units (t)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
+
+ // Nested block based table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Last one
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;}",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Mismatch curly braces
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={{{block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ // Unexpected chars after closing curly brace
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa;"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ // Invalid block based table option
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={xx_block_size=4;}",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=true",
+ &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=false",
+ &new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=junk",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+
+ // Nested plain table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+
+ // memtable factory
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "memtable=skip_list:10;arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
+}
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported
+TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
+ BlockBasedTableOptions table_opt;
+ BlockBasedTableOptions new_opt;
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kHashSearch;"
+ "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
+ "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
+ "block_size_deviation=8;block_restart_interval=4;"
+ "format_version=5;whole_key_filtering=1;"
+ "filter_policy=bloomfilter:4.567:false;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
+ ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
+ ASSERT_TRUE(new_opt.hash_index_allow_collision);
+ ASSERT_TRUE(new_opt.no_block_cache);
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL);
+ ASSERT_EQ(new_opt.block_size, 1024UL);
+ ASSERT_EQ(new_opt.block_size_deviation, 8);
+ ASSERT_EQ(new_opt.block_restart_interval, 4);
+ ASSERT_EQ(new_opt.format_version, 5U);
+ ASSERT_EQ(new_opt.whole_key_filtering, true);
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ const BloomFilterPolicy& bfp =
+ dynamic_cast<const BloomFilterPolicy&>(*new_opt.filter_policy);
+ EXPECT_EQ(bfp.GetMillibitsPerKey(), 4567);
+ EXPECT_EQ(bfp.GetWholeBitsPerKey(), 5);
+
+ // unknown option
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
+ "bad_option=1",
+ &new_opt));
+ ASSERT_EQ(static_cast<bool>(table_opt.cache_index_and_filter_blocks),
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized index type
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized checksum type
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;checksum=kxxHashXX",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized filter policy name
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;"
+ "filter_policy=bloomfilterxx:4:true",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
+
+ // unrecognized filter policy config
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;"
+ "filter_policy=bloomfilter:4",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
+
+ // Check block cache options are overwritten when specified
+ // in new format as a struct.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache_compressed)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set only block cache capacity. Check other values are
+ // reset to default values.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=2M};"
+ "block_cache_compressed={capacity=2M}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache)->GetNumShardBits(),
+ GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache_compressed)->GetNumShardBits(),
+ GetDefaultCacheShardBits(
+ new_opt.block_cache_compressed->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ table_opt,
+ "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={num_shard_bits=5;"
+ "high_pri_pool_ratio=0.0;}",
+ &new_opt));
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache)->GetNumShardBits(), 5);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache_compressed)->GetNumShardBits(), 5);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.0);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
+ new_opt.block_cache_compressed)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+}
+#endif // !ROCKSDB_LITE
+
+
+#ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported
+TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
+ PlainTableOptions table_opt;
+ PlainTableOptions new_opt;
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
+ "full_scan_mode=true;store_index_in_file=true",
+ &new_opt));
+ ASSERT_EQ(new_opt.user_key_len, 66u);
+ ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
+ ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
+ ASSERT_EQ(new_opt.index_sparseness, 8);
+ ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
+ ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
+ ASSERT_TRUE(new_opt.full_scan_mode);
+ ASSERT_TRUE(new_opt.store_index_in_file);
+
+ // unknown option
+ ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "bad_option=1",
+ &new_opt));
+
+ // unrecognized EncodingType
+ ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "encoding_type=kPrefixXX",
+ &new_opt));
+}
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported
+TEST_F(OptionsTest, GetMemTableRepFactoryFromString) {
+ std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr;
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory));
+ ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000",
+ &new_mem_factory));
+ ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist",
+ &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000",
+ &new_mem_factory));
+ ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory));
+ ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory));
+ // CuckooHash memtable is already removed.
+ ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory));
+
+ ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory));
+}
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
+TEST_F(OptionsTest, GetOptionsFromStringTest) {
+ Options base_options, new_options;
+ base_options.write_buffer_size = 20;
+ base_options.min_write_buffer_number_to_merge = 15;
+ BlockBasedTableOptions block_based_table_options;
+ block_based_table_options.cache_index_and_filter_blocks = true;
+ base_options.table_factory.reset(
+ NewBlockBasedTableFactory(block_based_table_options));
+
+ // Register an Env with object registry.
+ const static char* kCustomEnvName = "CustomEnv";
+ class CustomEnv : public EnvWrapper {
+ public:
+ explicit CustomEnv(Env* _target) : EnvWrapper(_target) {}
+ };
+
+ ObjectLibrary::Default()->Register<Env>(
+ kCustomEnvName,
+ [](const std::string& /*name*/, std::unique_ptr<Env>* /*env_guard*/,
+ std::string* /* errmsg */) {
+ static CustomEnv env(Env::Default());
+ return &env;
+ });
+
+ ASSERT_OK(GetOptionsFromString(
+ base_options,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;"
+ "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files="
+ "1;"
+ "rate_limiter_bytes_per_sec=1024;env=CustomEnv",
+ &new_options));
+
+ ASSERT_EQ(new_options.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_options.compression_opts.level, 5);
+ ASSERT_EQ(new_options.compression_opts.strategy, 6);
+ ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.enabled, false);
+ ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption);
+ ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_options.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
+ ASSERT_EQ(new_options.write_buffer_size, 10U);
+ ASSERT_EQ(new_options.max_write_buffer_number, 16);
+ BlockBasedTableOptions new_block_based_table_options =
+ dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get())
+ ->table_options();
+ ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20);
+ ASSERT_EQ(new_block_based_table_options.block_size, 4U);
+ // don't overwrite block based table options
+ ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks);
+
+ ASSERT_EQ(new_options.create_if_missing, true);
+ ASSERT_EQ(new_options.max_open_files, 1);
+ ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
+ Env* newEnv = new_options.env;
+ ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv));
+ ASSERT_EQ(newEnv, new_options.env);
+}
+
+TEST_F(OptionsTest, DBOptionsSerialization) {
+ Options base_options, new_options;
+ Random rnd(301);
+
+ // Phase 1: Make big change in base_options
+ test::RandomInitDBOptions(&base_options, &rnd);
+
+ // Phase 2: obtain a string from base_option
+ std::string base_options_file_content;
+ ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options));
+
+ // Phase 3: Set new_options from the derived string and expect
+ // new_options == base_options
+ ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content,
+ &new_options));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options));
+}
+
+TEST_F(OptionsTest, OptionsComposeDecompose) {
+ // build an Options from DBOptions + CFOptions, then decompose it to verify
+ // we get same constituent options.
+ DBOptions base_db_opts;
+ ColumnFamilyOptions base_cf_opts;
+
+ Random rnd(301);
+ test::RandomInitDBOptions(&base_db_opts, &rnd);
+ test::RandomInitCFOptions(&base_cf_opts, base_db_opts, &rnd);
+
+ Options base_opts(base_db_opts, base_cf_opts);
+ DBOptions new_db_opts(base_opts);
+ ColumnFamilyOptions new_cf_opts(base_opts);
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opts, new_db_opts));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opts, new_cf_opts));
+ delete new_cf_opts.compaction_filter;
+}
+
+TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
+ Options options;
+ ColumnFamilyOptions base_opt, new_opt;
+ Random rnd(302);
+ // Phase 1: randomly assign base_opt
+ // custom type options
+ test::RandomInitCFOptions(&base_opt, options, &rnd);
+
+ // Phase 2: obtain a string from base_opt
+ std::string base_options_file_content;
+ ASSERT_OK(
+ GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt));
+
+ // Phase 3: Set new_opt from the derived string and expect
+ // new_opt == base_opt
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ ColumnFamilyOptions(), base_options_file_content, &new_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt));
+ if (base_opt.compaction_filter) {
+ delete base_opt.compaction_filter;
+ }
+}
+
+#endif // !ROCKSDB_LITE
+
+Status StringToMap(
+ const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map);
+
+#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
+TEST_F(OptionsTest, StringToMapTest) {
+ std::unordered_map<std::string, std::string> opts_map;
+ // Regular options
+ ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "v2");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ // Value with '='
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "=v1");
+ ASSERT_EQ(opts_map["k2"], "v2=");
+ // Overwrriten option
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v2");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ // Empty value
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
+ ASSERT_EQ(opts_map["k4"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
+ ASSERT_EQ(opts_map["k4"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
+ ASSERT_EQ(opts_map["k3"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
+ ASSERT_EQ(opts_map["k3"], "");
+ // Regular nested options
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ // Multi-level nested options
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};"
+ "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4",
+ &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}");
+ ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}");
+ ASSERT_EQ(opts_map["k4"], "v4");
+ // Garbage inside curly braces
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4",
+ &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "dfad=");
+ ASSERT_EQ(opts_map["k3"], "=");
+ ASSERT_EQ(opts_map["k4"], "v4");
+ // Empty nested options
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}");
+ // With random spaces
+ opts_map.clear();
+ ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; "
+ "k3={ { } }; k4= v4 ",
+ &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}");
+ ASSERT_EQ(opts_map["k3"], "{ }");
+ ASSERT_EQ(opts_map["k4"], "v4");
+
+ // Empty key
+ ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map));
+ ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map));
+ // Mismatch curly braces
+ ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map));
+ // However this is valid!
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "}");
+ ASSERT_EQ(opts_map["k3"], "v3");
+
+ // Invalid chars after closing curly brace
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map));
+}
+#endif // ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
+TEST_F(OptionsTest, StringToMapRandomTest) {
+ std::unordered_map<std::string, std::string> opts_map;
+ // Make sure segfault is not hit by semi-random strings
+
+ std::vector<std::string> bases = {
+ "a={aa={};tt={xxx={}}};c=defff",
+ "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}",
+ "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"};
+
+ for (std::string base : bases) {
+ for (int rand_seed = 301; rand_seed < 401; rand_seed++) {
+ Random rnd(rand_seed);
+ for (int attempt = 0; attempt < 10; attempt++) {
+ std::string str = base;
+ // Replace random position to space
+ size_t pos = static_cast<size_t>(
+ rnd.Uniform(static_cast<int>(base.size())));
+ str[pos] = ' ';
+ Status s = StringToMap(str, &opts_map);
+ ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+ opts_map.clear();
+ }
+ }
+ }
+
+ // Random Construct a string
+ std::vector<char> chars = {'{', '}', ' ', '=', ';', 'c'};
+ for (int rand_seed = 301; rand_seed < 1301; rand_seed++) {
+ Random rnd(rand_seed);
+ int len = rnd.Uniform(30);
+ std::string str = "";
+ for (int attempt = 0; attempt < len; attempt++) {
+ // Add a random character
+ size_t pos = static_cast<size_t>(
+ rnd.Uniform(static_cast<int>(chars.size())));
+ str.append(1, chars[pos]);
+ }
+ Status s = StringToMap(str, &opts_map);
+ ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+ s = StringToMap("name=" + str, &opts_map);
+ ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+ opts_map.clear();
+ }
+}
+
+TEST_F(OptionsTest, GetStringFromCompressionType) {
+ std::string res;
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression));
+ ASSERT_EQ(res, "kNoCompression");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression));
+ ASSERT_EQ(res, "kSnappyCompression");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption));
+ ASSERT_EQ(res, "kDisableCompressionOption");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression));
+ ASSERT_EQ(res, "kLZ4Compression");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression));
+ ASSERT_EQ(res, "kZlibCompression");
+
+ ASSERT_NOK(
+ GetStringFromCompressionType(&res, static_cast<CompressionType>(-10)));
+}
+#endif // !ROCKSDB_LITE
+
+TEST_F(OptionsTest, ConvertOptionsTest) {
+ LevelDBOptions leveldb_opt;
+ Options converted_opt = ConvertOptions(leveldb_opt);
+
+ ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing);
+ ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists);
+ ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks);
+ ASSERT_EQ(converted_opt.env, leveldb_opt.env);
+ ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log);
+ ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size);
+ ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files);
+ ASSERT_EQ(converted_opt.compression, leveldb_opt.compression);
+
+ std::shared_ptr<TableFactory> tb_guard = converted_opt.table_factory;
+ BlockBasedTableFactory* table_factory =
+ dynamic_cast<BlockBasedTableFactory*>(converted_opt.table_factory.get());
+
+ ASSERT_TRUE(table_factory != nullptr);
+
+ const BlockBasedTableOptions table_opt = table_factory->table_options();
+
+ ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20);
+ ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size);
+ ASSERT_EQ(table_opt.block_restart_interval,
+ leveldb_opt.block_restart_interval);
+ ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy);
+}
+
+#ifndef ROCKSDB_LITE
+class OptionsParserTest : public testing::Test {
+ public:
+ OptionsParserTest() {
+ env_.reset(new test::StringEnv(Env::Default()));
+ fs_.reset(new LegacyFileSystemWrapper(env_.get()));
+ }
+
+ protected:
+ std::unique_ptr<test::StringEnv> env_;
+ std::unique_ptr<LegacyFileSystemWrapper> fs_;
+};
+
+TEST_F(OptionsParserTest, Comment) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[ DBOptions ]\n"
+ " # note that we don't support space around \"=\"\n"
+ " max_open_files=12345;\n"
+ " max_background_flushes=301 # comment after a statement is fine\n"
+ " # max_background_flushes=1000 # this line would be ignored\n"
+ " # max_background_compactions=2000 # so does this one\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"default\"] # column family must be specified\n"
+ " # in the correct order\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_OK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt));
+ ASSERT_EQ(parser.NumColumnFamilies(), 1U);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ *parser.GetCFOptions("default"), cf_opt));
+}
+
+TEST_F(OptionsParserTest, ExtraSpace) {
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[ Version ]\n"
+ " rocksdb_version = 3.14.0 \n"
+ " options_file_version=1 # some comment\n"
+ "[DBOptions ] # some comment\n"
+ "max_open_files=12345 \n"
+ " max_background_flushes = 301 \n"
+ " max_total_wal_size = 1024 # keep_log_file_num=1000\n"
+ " [CFOptions \"default\" ]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_OK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, MissingDBOptions) {
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[CFOptions \"default\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, DoubleDBOptions) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[DBOptions]\n"
+ "[CFOptions \"default\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, NoDefaultCFOptions) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"something_else\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"something_else\"]\n"
+ " # if a section is blank, we will use the default\n"
+ "[CFOptions \"default\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, DuplicateCFOptions) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"default\"]\n"
+ "[CFOptions \"something_else\"]\n"
+ "[CFOptions \"something_else\"]\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, IgnoreUnknownOptions) {
+ for (int case_id = 0; case_id < 5; case_id++) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string version_string;
+ bool should_ignore = true;
+ if (case_id == 0) {
+ // same version
+ should_ignore = false;
+ version_string =
+ ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) + ".0";
+ } else if (case_id == 1) {
+ // higher minor version
+ should_ignore = true;
+ version_string =
+ ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR + 1) + ".0";
+ } else if (case_id == 2) {
+ // higher major version.
+ should_ignore = true;
+ version_string = ToString(ROCKSDB_MAJOR + 1) + ".0.0";
+ } else if (case_id == 3) {
+ // lower minor version
+#if ROCKSDB_MINOR == 0
+ continue;
+#else
+ version_string =
+ ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR - 1) + ".0";
+ should_ignore = false;
+#endif
+ } else {
+ // lower major version
+ should_ignore = false;
+ version_string =
+ ToString(ROCKSDB_MAJOR - 1) + "." + ToString(ROCKSDB_MINOR) + ".0";
+ }
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=" +
+ version_string +
+ "\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ " unknown_db_option1=321\n"
+ " unknown_db_option2=false\n"
+ "[CFOptions \"default\"]\n"
+ " unknown_cf_option1=hello\n"
+ "[CFOptions \"something_else\"]\n"
+ " unknown_cf_option2=world\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ env_->DeleteFile(kTestFileName);
+ env_->WriteToNewFile(kTestFileName, options_file_content);
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), false,
+ 4096 /* readahead_size */));
+ if (should_ignore) {
+ ASSERT_OK(parser.Parse(kTestFileName, fs_.get(),
+ true /* ignore_unknown_options */,
+ 4096 /* readahead_size */));
+ } else {
+ ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(),
+ true /* ignore_unknown_options */,
+ 4096 /* readahead_size */));
+ }
+ }
+}
+
+TEST_F(OptionsParserTest, ParseVersion) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string file_template =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.13.1\n"
+ " options_file_version=%s\n"
+ "[DBOptions]\n"
+ "[CFOptions \"default\"]\n";
+ const int kLength = 1000;
+ char buffer[kLength];
+ RocksDBOptionsParser parser;
+
+ const std::vector<std::string> invalid_versions = {
+ "a.b.c", "3.2.2b", "3.-12", "3. 1", // only digits and dots are allowed
+ "1.2.3.4",
+ "1.2.3" // can only contains at most one dot.
+ "0", // options_file_version must be at least one
+ "3..2",
+ ".", ".1.2", // must have at least one digit before each dot
+ "1.2.", "1.", "2.34."}; // must have at least one digit after each dot
+ for (auto iv : invalid_versions) {
+ snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str());
+
+ parser.Reset();
+ env_->WriteToNewFile(iv, buffer);
+ ASSERT_NOK(parser.Parse(iv, fs_.get(), false, 0 /* readahead_size */));
+ }
+
+ const std::vector<std::string> valid_versions = {
+ "1.232", "100", "3.12", "1", "12.3 ", " 1.25 "};
+ for (auto vv : valid_versions) {
+ snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str());
+ parser.Reset();
+ env_->WriteToNewFile(vv, buffer);
+ ASSERT_OK(parser.Parse(vv, fs_.get(), false, 0 /* readahead_size */));
+ }
+}
+
+void VerifyCFPointerTypedOptions(
+ ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
+ const std::unordered_map<std::string, std::string>* new_cf_opt_map) {
+ std::string name_buffer;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+ new_cf_opt_map));
+
+ // change the name of merge operator back-and-forth
+ {
+ auto* merge_operator = dynamic_cast<test::ChanglingMergeOperator*>(
+ base_cf_opt->merge_operator.get());
+ if (merge_operator != nullptr) {
+ name_buffer = merge_operator->Name();
+ // change the name and expect non-ok status
+ merge_operator->SetName("some-other-name");
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // change the name back and expect ok status
+ merge_operator->SetName(name_buffer);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+ new_cf_opt_map));
+ }
+ }
+
+ // change the name of the compaction filter factory back-and-forth
+ {
+ auto* compaction_filter_factory =
+ dynamic_cast<test::ChanglingCompactionFilterFactory*>(
+ base_cf_opt->compaction_filter_factory.get());
+ if (compaction_filter_factory != nullptr) {
+ name_buffer = compaction_filter_factory->Name();
+ // change the name and expect non-ok status
+ compaction_filter_factory->SetName("some-other-name");
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // change the name back and expect ok status
+ compaction_filter_factory->SetName(name_buffer);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+ new_cf_opt_map));
+ }
+ }
+
+ // test by setting compaction_filter to nullptr
+ {
+ auto* tmp_compaction_filter = base_cf_opt->compaction_filter;
+ if (tmp_compaction_filter != nullptr) {
+ base_cf_opt->compaction_filter = nullptr;
+ // set compaction_filter to nullptr and expect non-ok status
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // set the value back and expect ok status
+ base_cf_opt->compaction_filter = tmp_compaction_filter;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+ new_cf_opt_map));
+ }
+ }
+
+ // test by setting table_factory to nullptr
+ {
+ auto tmp_table_factory = base_cf_opt->table_factory;
+ if (tmp_table_factory != nullptr) {
+ base_cf_opt->table_factory.reset();
+ // set table_factory to nullptr and expect non-ok status
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // set the value back and expect ok status
+ base_cf_opt->table_factory = tmp_table_factory;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+ new_cf_opt_map));
+ }
+ }
+
+ // test by setting memtable_factory to nullptr
+ {
+ auto tmp_memtable_factory = base_cf_opt->memtable_factory;
+ if (tmp_memtable_factory != nullptr) {
+ base_cf_opt->memtable_factory.reset();
+ // set memtable_factory to nullptr and expect non-ok status
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // set the value back and expect ok status
+ base_cf_opt->memtable_factory = tmp_memtable_factory;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+ new_cf_opt_map));
+ }
+ }
+}
+
+TEST_F(OptionsParserTest, Readahead) {
+ DBOptions base_db_opt;
+ std::vector<ColumnFamilyOptions> base_cf_opts;
+ base_cf_opts.emplace_back();
+ base_cf_opts.emplace_back();
+
+ std::string one_mb_string = std::string(1024 * 1024, 'x');
+ std::vector<std::string> cf_names = {"default", one_mb_string};
+ const std::string kOptionsFileName = "test-persisted-options.ini";
+
+ ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
+ kOptionsFileName, fs_.get()));
+
+ uint64_t file_size = 0;
+ ASSERT_OK(env_->GetFileSize(kOptionsFileName, &file_size));
+ assert(file_size > 0);
+
+ RocksDBOptionsParser parser;
+
+ env_->num_seq_file_read_ = 0;
+ size_t readahead_size = 128 * 1024;
+
+ ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
+ ASSERT_EQ(env_->num_seq_file_read_.load(),
+ (file_size - 1) / readahead_size + 1);
+
+ env_->num_seq_file_read_.store(0);
+ readahead_size = 1024 * 1024;
+ ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
+ ASSERT_EQ(env_->num_seq_file_read_.load(),
+ (file_size - 1) / readahead_size + 1);
+
+ // Tiny readahead. 8 KB is read each time.
+ env_->num_seq_file_read_.store(0);
+ ASSERT_OK(
+ parser.Parse(kOptionsFileName, fs_.get(), false, 1 /* readahead_size */));
+ ASSERT_GE(env_->num_seq_file_read_.load(), file_size / (8 * 1024));
+ ASSERT_LT(env_->num_seq_file_read_.load(), file_size / (8 * 1024) * 2);
+
+ // Disable readahead means 512KB readahead.
+ env_->num_seq_file_read_.store(0);
+ ASSERT_OK(
+ parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
+ ASSERT_GE(env_->num_seq_file_read_.load(),
+ (file_size - 1) / (512 * 1024) + 1);
+}
+
+TEST_F(OptionsParserTest, DumpAndParse) {
+ DBOptions base_db_opt;
+ std::vector<ColumnFamilyOptions> base_cf_opts;
+ std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3",
+ "c:f:4:4:4"
+ "p\\i\\k\\a\\chu\\\\\\",
+ "###rocksdb#1-testcf#2###"};
+ const int num_cf = static_cast<int>(cf_names.size());
+ Random rnd(302);
+ test::RandomInitDBOptions(&base_db_opt, &rnd);
+ base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG";
+
+ BlockBasedTableOptions special_bbto;
+ special_bbto.cache_index_and_filter_blocks = true;
+ special_bbto.block_size = 999999;
+
+ for (int c = 0; c < num_cf; ++c) {
+ ColumnFamilyOptions cf_opt;
+ Random cf_rnd(0xFB + c);
+ test::RandomInitCFOptions(&cf_opt, base_db_opt, &cf_rnd);
+ if (c < 4) {
+ cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c));
+ }
+ if (c < 3) {
+ cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c));
+ } else if (c == 4) {
+ cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto));
+ }
+ base_cf_opts.emplace_back(cf_opt);
+ }
+
+ const std::string kOptionsFileName = "test-persisted-options.ini";
+ ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
+ kOptionsFileName, fs_.get()));
+
+ RocksDBOptionsParser parser;
+ ASSERT_OK(
+ parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
+
+ // Make sure block-based table factory options was deserialized correctly
+ std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory;
+ ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name()));
+ const BlockBasedTableOptions& parsed_bbto =
+ static_cast<BlockBasedTableFactory*>(ttf.get())->table_options();
+ ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size);
+ ASSERT_EQ(special_bbto.cache_index_and_filter_blocks,
+ parsed_bbto.cache_index_and_filter_blocks);
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
+
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt));
+ for (int c = 0; c < num_cf; ++c) {
+ const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
+ ASSERT_NE(cf_opt, nullptr);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c))));
+ }
+
+ // Further verify pointer-typed options
+ for (int c = 0; c < num_cf; ++c) {
+ const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
+ ASSERT_NE(cf_opt, nullptr);
+ VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt,
+ &(parser.cf_opt_maps()->at(c)));
+ }
+
+ ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr);
+
+ base_db_opt.max_open_files++;
+ ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
+
+ for (int c = 0; c < num_cf; ++c) {
+ if (base_cf_opts[c].compaction_filter) {
+ delete base_cf_opts[c].compaction_filter;
+ }
+ }
+}
+
+TEST_F(OptionsParserTest, DifferentDefault) {
+ const std::string kOptionsFileName = "test-persisted-options.ini";
+
+ ColumnFamilyOptions cf_level_opts;
+ ASSERT_EQ(CompactionPri::kMinOverlappingRatio, cf_level_opts.compaction_pri);
+ cf_level_opts.OptimizeLevelStyleCompaction();
+
+ ColumnFamilyOptions cf_univ_opts;
+ cf_univ_opts.OptimizeUniversalStyleCompaction();
+
+ ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"},
+ {cf_level_opts, cf_univ_opts},
+ kOptionsFileName, fs_.get()));
+
+ RocksDBOptionsParser parser;
+ ASSERT_OK(
+ parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
+
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults();
+ ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
+ ASSERT_EQ(5000, old_default_opts.max_open_files);
+ ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ ASSERT_EQ(WALRecoveryMode::kTolerateCorruptedTailRecords,
+ old_default_opts.wal_recovery_mode);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(4, 6);
+ ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
+ ASSERT_EQ(5000, old_default_opts.max_open_files);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(4, 7);
+ ASSERT_NE(10 * 1048576, old_default_opts.max_bytes_for_level_base);
+ ASSERT_NE(4, old_default_opts.table_cache_numshardbits);
+ ASSERT_EQ(5000, old_default_opts.max_open_files);
+ ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ }
+ {
+ ColumnFamilyOptions old_default_cf_opts;
+ old_default_cf_opts.OldDefaults();
+ ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(4 << 20, old_default_cf_opts.write_buffer_size);
+ ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(0, old_default_cf_opts.soft_pending_compaction_bytes_limit);
+ ASSERT_EQ(0, old_default_cf_opts.hard_pending_compaction_bytes_limit);
+ ASSERT_EQ(CompactionPri::kByCompensatedSize,
+ old_default_cf_opts.compaction_pri);
+ }
+ {
+ ColumnFamilyOptions old_default_cf_opts;
+ old_default_cf_opts.OldDefaults(4, 6);
+ ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(CompactionPri::kByCompensatedSize,
+ old_default_cf_opts.compaction_pri);
+ }
+ {
+ ColumnFamilyOptions old_default_cf_opts;
+ old_default_cf_opts.OldDefaults(4, 7);
+ ASSERT_NE(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(CompactionPri::kByCompensatedSize,
+ old_default_cf_opts.compaction_pri);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(5, 1);
+ ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(5, 2);
+ ASSERT_EQ(16 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ ASSERT_TRUE(old_default_opts.compaction_pri ==
+ CompactionPri::kByCompensatedSize);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(5, 18);
+ ASSERT_TRUE(old_default_opts.compaction_pri ==
+ CompactionPri::kByCompensatedSize);
+ }
+
+ Options small_opts;
+ small_opts.OptimizeForSmallDb();
+ ASSERT_EQ(2 << 20, small_opts.write_buffer_size);
+ ASSERT_EQ(5000, small_opts.max_open_files);
+}
+
+class OptionsSanityCheckTest : public OptionsParserTest {
+ public:
+ OptionsSanityCheckTest() {}
+
+ protected:
+ Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts,
+ OptionsSanityCheckLevel level) {
+ return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, fs_.get(),
+ level);
+ }
+
+ Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) {
+ Status s = env_->DeleteFile(kOptionsFileName);
+ if (!s.ok()) {
+ return s;
+ }
+ return PersistRocksDBOptions(DBOptions(), {"default"}, {cf_opts},
+ kOptionsFileName, fs_.get());
+ }
+
+ const std::string kOptionsFileName = "OPTIONS";
+};
+
+TEST_F(OptionsSanityCheckTest, SanityCheck) {
+ ColumnFamilyOptions opts;
+ Random rnd(301);
+
+ // default ColumnFamilyOptions
+ {
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ }
+
+ // prefix_extractor
+ {
+ // Okay to change prefix_extractor form nullptr to non-nullptr
+ ASSERT_EQ(opts.prefix_extractor.get(), nullptr);
+ opts.prefix_extractor.reset(NewCappedPrefixTransform(10));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+
+ // use same prefix extractor but with different parameter
+ opts.prefix_extractor.reset(NewCappedPrefixTransform(15));
+ // expect pass only in kSanityLevelLooselyCompatible
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // repeat the test with FixedPrefixTransform
+ opts.prefix_extractor.reset(NewFixedPrefixTransform(10));
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // persist the change of prefix_extractor
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+
+ // use same prefix extractor but with different parameter
+ opts.prefix_extractor.reset(NewFixedPrefixTransform(15));
+ // expect pass only in kSanityLevelLooselyCompatible
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // Change prefix extractor from non-nullptr to nullptr
+ opts.prefix_extractor.reset();
+ // expect pass as it's safe to change prefix_extractor
+ // from non-null to null
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ }
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+
+ // table_factory
+ {
+ for (int tb = 0; tb <= 2; ++tb) {
+ // change the table factory
+ opts.table_factory.reset(test::RandomTableFactory(&rnd, tb));
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ }
+ }
+
+ // merge_operator
+ {
+ // Test when going from nullptr -> merge operator
+ opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+
+ for (int test = 0; test < 5; ++test) {
+ // change the merge operator
+ opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ }
+
+ // Test when going from merge operator -> nullptr
+ opts.merge_operator = nullptr;
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ }
+
+ // compaction_filter
+ {
+ for (int test = 0; test < 5; ++test) {
+ // change the compaction filter
+ opts.compaction_filter = test::RandomCompactionFilter(&rnd);
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ delete opts.compaction_filter;
+ opts.compaction_filter = nullptr;
+ }
+ }
+
+ // compaction_filter_factory
+ {
+ for (int test = 0; test < 5; ++test) {
+ // change the compaction filter factory
+ opts.compaction_filter_factory.reset(
+ test::RandomCompactionFilterFactory(&rnd));
+ ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ }
+ }
+}
+
+namespace {
+bool IsEscapedString(const std::string& str) {
+ for (size_t i = 0; i < str.size(); ++i) {
+ if (str[i] == '\\') {
+ // since we already handle those two consecutive '\'s in
+ // the next if-then branch, any '\' appear at the end
+ // of an escaped string in such case is not valid.
+ if (i == str.size() - 1) {
+ return false;
+ }
+ if (str[i + 1] == '\\') {
+ // if there're two consecutive '\'s, skip the second one.
+ i++;
+ continue;
+ }
+ switch (str[i + 1]) {
+ case ':':
+ case '\\':
+ case '#':
+ continue;
+ default:
+ // if true, '\' together with str[i + 1] is not a valid escape.
+ if (UnescapeChar(str[i + 1]) == str[i + 1]) {
+ return false;
+ }
+ }
+ } else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) {
+ return false;
+ }
+ }
+ return true;
+}
+} // namespace
+
+TEST_F(OptionsParserTest, IntegerParsing) {
+ ASSERT_EQ(ParseUint64("18446744073709551615"), 18446744073709551615U);
+ ASSERT_EQ(ParseUint32("4294967295"), 4294967295U);
+ ASSERT_EQ(ParseSizeT("18446744073709551615"), 18446744073709551615U);
+ ASSERT_EQ(ParseInt64("9223372036854775807"), 9223372036854775807);
+ ASSERT_EQ(ParseInt64("-9223372036854775808"), port::kMinInt64);
+ ASSERT_EQ(ParseInt32("2147483647"), 2147483647);
+ ASSERT_EQ(ParseInt32("-2147483648"), port::kMinInt32);
+ ASSERT_EQ(ParseInt("-32767"), -32767);
+ ASSERT_EQ(ParseDouble("-1.234567"), -1.234567);
+}
+
+TEST_F(OptionsParserTest, EscapeOptionString) {
+ ASSERT_EQ(UnescapeOptionString(
+ "This is a test string with \\# \\: and \\\\ escape chars."),
+ "This is a test string with # : and \\ escape chars.");
+
+ ASSERT_EQ(
+ EscapeOptionString("This is a test string with # : and \\ escape chars."),
+ "This is a test string with \\# \\: and \\\\ escape chars.");
+
+ std::string readible_chars =
+ "A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU"
+ "YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>"
+ "<MNBVCXZ\\\" should be okay to \\#\\\\\\:\\#\\#\\#\\ "
+ "be serialized and deserialized";
+
+ std::string escaped_string = EscapeOptionString(readible_chars);
+ ASSERT_TRUE(IsEscapedString(escaped_string));
+ // This two transformations should be canceled and should output
+ // the original input.
+ ASSERT_EQ(UnescapeOptionString(escaped_string), readible_chars);
+
+ std::string all_chars;
+ for (unsigned char c = 0;; ++c) {
+ all_chars += c;
+ if (c == 255) {
+ break;
+ }
+ }
+ escaped_string = EscapeOptionString(all_chars);
+ ASSERT_TRUE(IsEscapedString(escaped_string));
+ ASSERT_EQ(UnescapeOptionString(escaped_string), all_chars);
+
+ ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
+ " A simple statement with a comment. # like this :)"),
+ "A simple statement with a comment.");
+
+ ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
+ "Escape \\# and # comment together ."),
+ "Escape \\# and");
+}
+#endif // !ROCKSDB_LITE
+} // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+#ifdef GFLAGS
+ ParseCommandLineFlags(&argc, &argv, true);
+#endif // GFLAGS
+ return RUN_ALL_TESTS();
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-17 20:02:18 +0000