summaryrefslogtreecommitdiffstats
path: root/src/rocksdb/options
diff options
context:
space:
mode:
Diffstat (limited to 'src/rocksdb/options')
-rw-r--r--src/rocksdb/options/cf_options.cc1166
-rw-r--r--src/rocksdb/options/cf_options.h344
-rw-r--r--src/rocksdb/options/configurable.cc767
-rw-r--r--src/rocksdb/options/configurable_helper.h187
-rw-r--r--src/rocksdb/options/configurable_test.cc881
-rw-r--r--src/rocksdb/options/configurable_test.h126
-rw-r--r--src/rocksdb/options/customizable.cc139
-rw-r--r--src/rocksdb/options/customizable_test.cc2255
-rw-r--r--src/rocksdb/options/db_options.cc1086
-rw-r--r--src/rocksdb/options/db_options.h156
-rw-r--r--src/rocksdb/options/options.cc735
-rw-r--r--src/rocksdb/options/options_helper.cc1478
-rw-r--r--src/rocksdb/options/options_helper.h122
-rw-r--r--src/rocksdb/options/options_parser.cc727
-rw-r--r--src/rocksdb/options/options_parser.h151
-rw-r--r--src/rocksdb/options/options_settable_test.cc621
-rw-r--r--src/rocksdb/options/options_test.cc5014
17 files changed, 15955 insertions, 0 deletions
diff --git a/src/rocksdb/options/cf_options.cc b/src/rocksdb/options/cf_options.cc
new file mode 100644
index 000000000..dbf0bf9b0
--- /dev/null
+++ b/src/rocksdb/options/cf_options.cc
@@ -0,0 +1,1166 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "options/cf_options.h"
+
+#include <cassert>
+#include <cinttypes>
+#include <limits>
+#include <string>
+
+#include "logging/logging.h"
+#include "options/configurable_helper.h"
+#include "options/db_options.h"
+#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "port/port.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/concurrent_task_limiter.h"
+#include "rocksdb/configurable.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/env.h"
+#include "rocksdb/file_system.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/table.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_type.h"
+#include "util/cast_util.h"
+
+// NOTE: in this file, many option flags that were deprecated
+// and removed from the rest of the code have to be kept here
+// and marked as kDeprecated in order to be able to read old
+// OPTIONS files.
+
+namespace ROCKSDB_NAMESPACE {
+
+#ifndef ROCKSDB_LITE
+static Status ParseCompressionOptions(const std::string& value,
+ const std::string& name,
+ CompressionOptions& compression_opts) {
+ const char kDelimiter = ':';
+ std::istringstream field_stream(value);
+ std::string field;
+
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument("unable to parse the specified CF option " +
+ name);
+ }
+ compression_opts.window_bits = ParseInt(field);
+
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument("unable to parse the specified CF option " +
+ name);
+ }
+ compression_opts.level = ParseInt(field);
+
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument("unable to parse the specified CF option " +
+ name);
+ }
+ compression_opts.strategy = ParseInt(field);
+
+ // max_dict_bytes is optional for backwards compatibility
+ if (!field_stream.eof()) {
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.max_dict_bytes = ParseInt(field);
+ }
+
+ // zstd_max_train_bytes is optional for backwards compatibility
+ if (!field_stream.eof()) {
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.zstd_max_train_bytes = ParseInt(field);
+ }
+
+ // parallel_threads is optional for backwards compatibility
+ if (!field_stream.eof()) {
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ // Since parallel_threads comes before enabled but was added optionally
+ // later, we need to check if this is the final token (meaning it is the
+ // enabled bit), or if there are more tokens (meaning this one is
+ // parallel_threads).
+ if (!field_stream.eof()) {
+ compression_opts.parallel_threads = ParseInt(field);
+ } else {
+ // parallel_threads is not serialized with this format, but enabled is
+ compression_opts.enabled = ParseBoolean("", field);
+ }
+ }
+
+ // enabled is optional for backwards compatibility
+ if (!field_stream.eof()) {
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.enabled = ParseBoolean("", field);
+ }
+
+ // max_dict_buffer_bytes is optional for backwards compatibility
+ if (!field_stream.eof()) {
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.max_dict_buffer_bytes = ParseUint64(field);
+ }
+
+ // use_zstd_dict_trainer is optional for backwards compatibility
+ if (!field_stream.eof()) {
+ if (!std::getline(field_stream, field, kDelimiter)) {
+ return Status::InvalidArgument(
+ "unable to parse the specified CF option " + name);
+ }
+ compression_opts.use_zstd_dict_trainer = ParseBoolean("", field);
+ }
+
+ if (!field_stream.eof()) {
+ return Status::InvalidArgument("unable to parse the specified CF option " +
+ name);
+ }
+ return Status::OK();
+}
+
+const std::string kOptNameBMCompOpts = "bottommost_compression_opts";
+const std::string kOptNameCompOpts = "compression_opts";
+
+// OptionTypeInfo map for CompressionOptions
+static std::unordered_map<std::string, OptionTypeInfo>
+ compression_options_type_info = {
+ {"window_bits",
+ {offsetof(struct CompressionOptions, window_bits), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"level",
+ {offsetof(struct CompressionOptions, level), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"strategy",
+ {offsetof(struct CompressionOptions, strategy), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"max_dict_bytes",
+ {offsetof(struct CompressionOptions, max_dict_bytes), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"zstd_max_train_bytes",
+ {offsetof(struct CompressionOptions, zstd_max_train_bytes),
+ OptionType::kUInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"parallel_threads",
+ {offsetof(struct CompressionOptions, parallel_threads),
+ OptionType::kUInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"enabled",
+ {offsetof(struct CompressionOptions, enabled), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"max_dict_buffer_bytes",
+ {offsetof(struct CompressionOptions, max_dict_buffer_bytes),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"use_zstd_dict_trainer",
+ {offsetof(struct CompressionOptions, use_zstd_dict_trainer),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+};
+
+static std::unordered_map<std::string, OptionTypeInfo>
+ fifo_compaction_options_type_info = {
+ {"max_table_files_size",
+ {offsetof(struct CompactionOptionsFIFO, max_table_files_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"age_for_warm",
+ {offsetof(struct CompactionOptionsFIFO, age_for_warm),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"ttl",
+ {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"allow_compaction",
+ {offsetof(struct CompactionOptionsFIFO, allow_compaction),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+};
+
+static std::unordered_map<std::string, OptionTypeInfo>
+ universal_compaction_options_type_info = {
+ {"size_ratio",
+ {offsetof(class CompactionOptionsUniversal, size_ratio),
+ OptionType::kUInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"min_merge_width",
+ {offsetof(class CompactionOptionsUniversal, min_merge_width),
+ OptionType::kUInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_merge_width",
+ {offsetof(class CompactionOptionsUniversal, max_merge_width),
+ OptionType::kUInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_size_amplification_percent",
+ {offsetof(class CompactionOptionsUniversal,
+ max_size_amplification_percent),
+ OptionType::kUInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"compression_size_percent",
+ {offsetof(class CompactionOptionsUniversal, compression_size_percent),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"stop_style",
+ {offsetof(class CompactionOptionsUniversal, stop_style),
+ OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"incremental",
+ {offsetof(class CompactionOptionsUniversal, incremental),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"allow_trivial_move",
+ {offsetof(class CompactionOptionsUniversal, allow_trivial_move),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}}};
+
+static std::unordered_map<std::string, OptionTypeInfo>
+ cf_mutable_options_type_info = {
+ {"report_bg_io_stats",
+ {offsetof(struct MutableCFOptions, report_bg_io_stats),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"disable_auto_compactions",
+ {offsetof(struct MutableCFOptions, disable_auto_compactions),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"filter_deletes",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"check_flush_compaction_key_order",
+ {offsetof(struct MutableCFOptions, check_flush_compaction_key_order),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"paranoid_file_checks",
+ {offsetof(struct MutableCFOptions, paranoid_file_checks),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"verify_checksums_in_compaction",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"soft_pending_compaction_bytes_limit",
+ {offsetof(struct MutableCFOptions,
+ soft_pending_compaction_bytes_limit),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"hard_pending_compaction_bytes_limit",
+ {offsetof(struct MutableCFOptions,
+ hard_pending_compaction_bytes_limit),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"hard_rate_limit",
+ {0, OptionType::kDouble, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"soft_rate_limit",
+ {0, OptionType::kDouble, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"max_compaction_bytes",
+ {offsetof(struct MutableCFOptions, max_compaction_bytes),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"ignore_max_compaction_bytes_for_input",
+ {offsetof(struct MutableCFOptions,
+ ignore_max_compaction_bytes_for_input),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"expanded_compaction_factor",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"level0_file_num_compaction_trigger",
+ {offsetof(struct MutableCFOptions, level0_file_num_compaction_trigger),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"level0_slowdown_writes_trigger",
+ {offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"level0_stop_writes_trigger",
+ {offsetof(struct MutableCFOptions, level0_stop_writes_trigger),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_grandparent_overlap_factor",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"max_write_buffer_number",
+ {offsetof(struct MutableCFOptions, max_write_buffer_number),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"source_compaction_factor",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"target_file_size_multiplier",
+ {offsetof(struct MutableCFOptions, target_file_size_multiplier),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"arena_block_size",
+ {offsetof(struct MutableCFOptions, arena_block_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"inplace_update_num_locks",
+ {offsetof(struct MutableCFOptions, inplace_update_num_locks),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_successive_merges",
+ {offsetof(struct MutableCFOptions, max_successive_merges),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"memtable_huge_page_size",
+ {offsetof(struct MutableCFOptions, memtable_huge_page_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"memtable_prefix_bloom_huge_page_tlb_size",
+ {0, OptionType::kSizeT, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"write_buffer_size",
+ {offsetof(struct MutableCFOptions, write_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"memtable_prefix_bloom_bits",
+ {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"memtable_prefix_bloom_size_ratio",
+ {offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio),
+ OptionType::kDouble, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"memtable_prefix_bloom_probes",
+ {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"memtable_whole_key_filtering",
+ {offsetof(struct MutableCFOptions, memtable_whole_key_filtering),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"min_partial_merge_operands",
+ {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"max_bytes_for_level_base",
+ {offsetof(struct MutableCFOptions, max_bytes_for_level_base),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"snap_refresh_nanos",
+ {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"max_bytes_for_level_multiplier",
+ {offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier),
+ OptionType::kDouble, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_bytes_for_level_multiplier_additional",
+ OptionTypeInfo::Vector<int>(
+ offsetof(struct MutableCFOptions,
+ max_bytes_for_level_multiplier_additional),
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
+ {0, OptionType::kInt})},
+ {"max_sequential_skip_in_iterations",
+ {offsetof(struct MutableCFOptions, max_sequential_skip_in_iterations),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"target_file_size_base",
+ {offsetof(struct MutableCFOptions, target_file_size_base),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"compression",
+ {offsetof(struct MutableCFOptions, compression),
+ OptionType::kCompressionType, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"prefix_extractor",
+ OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
+ offsetof(struct MutableCFOptions, prefix_extractor),
+ OptionVerificationType::kByNameAllowNull,
+ (OptionTypeFlags::kMutable | OptionTypeFlags::kAllowNull))},
+ {"compaction_options_fifo",
+ OptionTypeInfo::Struct(
+ "compaction_options_fifo", &fifo_compaction_options_type_info,
+ offsetof(struct MutableCFOptions, compaction_options_fifo),
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable)
+ .SetParseFunc([](const ConfigOptions& opts,
+ const std::string& name, const std::string& value,
+ void* addr) {
+ // This is to handle backward compatibility, where
+ // compaction_options_fifo could be assigned a single scalar
+ // value, say, like "23", which would be assigned to
+ // max_table_files_size.
+ if (name == "compaction_options_fifo" &&
+ value.find("=") == std::string::npos) {
+ // Old format. Parse just a single uint64_t value.
+ auto options = static_cast<CompactionOptionsFIFO*>(addr);
+ options->max_table_files_size = ParseUint64(value);
+ return Status::OK();
+ } else {
+ return OptionTypeInfo::ParseStruct(
+ opts, "compaction_options_fifo",
+ &fifo_compaction_options_type_info, name, value, addr);
+ }
+ })},
+ {"compaction_options_universal",
+ OptionTypeInfo::Struct(
+ "compaction_options_universal",
+ &universal_compaction_options_type_info,
+ offsetof(struct MutableCFOptions, compaction_options_universal),
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable)},
+ {"ttl",
+ {offsetof(struct MutableCFOptions, ttl), OptionType::kUInt64T,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"periodic_compaction_seconds",
+ {offsetof(struct MutableCFOptions, periodic_compaction_seconds),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"bottommost_temperature",
+ {0, OptionType::kTemperature, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"last_level_temperature",
+ {offsetof(struct MutableCFOptions, last_level_temperature),
+ OptionType::kTemperature, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"enable_blob_files",
+ {offsetof(struct MutableCFOptions, enable_blob_files),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"min_blob_size",
+ {offsetof(struct MutableCFOptions, min_blob_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"blob_file_size",
+ {offsetof(struct MutableCFOptions, blob_file_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"blob_compression_type",
+ {offsetof(struct MutableCFOptions, blob_compression_type),
+ OptionType::kCompressionType, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"enable_blob_garbage_collection",
+ {offsetof(struct MutableCFOptions, enable_blob_garbage_collection),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"blob_garbage_collection_age_cutoff",
+ {offsetof(struct MutableCFOptions, blob_garbage_collection_age_cutoff),
+ OptionType::kDouble, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"blob_garbage_collection_force_threshold",
+ {offsetof(struct MutableCFOptions,
+ blob_garbage_collection_force_threshold),
+ OptionType::kDouble, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"blob_compaction_readahead_size",
+ {offsetof(struct MutableCFOptions, blob_compaction_readahead_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"blob_file_starting_level",
+ {offsetof(struct MutableCFOptions, blob_file_starting_level),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"prepopulate_blob_cache",
+ OptionTypeInfo::Enum<PrepopulateBlobCache>(
+ offsetof(struct MutableCFOptions, prepopulate_blob_cache),
+ &prepopulate_blob_cache_string_map, OptionTypeFlags::kMutable)},
+ {"sample_for_compression",
+ {offsetof(struct MutableCFOptions, sample_for_compression),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"bottommost_compression",
+ {offsetof(struct MutableCFOptions, bottommost_compression),
+ OptionType::kCompressionType, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"compression_per_level",
+ OptionTypeInfo::Vector<CompressionType>(
+ offsetof(struct MutableCFOptions, compression_per_level),
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
+ {0, OptionType::kCompressionType})},
+ {"experimental_mempurge_threshold",
+ {offsetof(struct MutableCFOptions, experimental_mempurge_threshold),
+ OptionType::kDouble, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"memtable_protection_bytes_per_key",
+ {offsetof(struct MutableCFOptions, memtable_protection_bytes_per_key),
+ OptionType::kUInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {kOptNameCompOpts,
+ OptionTypeInfo::Struct(
+ kOptNameCompOpts, &compression_options_type_info,
+ offsetof(struct MutableCFOptions, compression_opts),
+ OptionVerificationType::kNormal,
+ (OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
+ [](const ConfigOptions& opts, const std::string& name,
+ const std::string& value, void* addr) {
+ // This is to handle backward compatibility, where
+ // compression_options was a ":" separated list.
+ if (name == kOptNameCompOpts &&
+ value.find("=") == std::string::npos) {
+ auto* compression = static_cast<CompressionOptions*>(addr);
+ return ParseCompressionOptions(value, name, *compression);
+ } else {
+ return OptionTypeInfo::ParseStruct(
+ opts, kOptNameCompOpts, &compression_options_type_info,
+ name, value, addr);
+ }
+ })},
+ {kOptNameBMCompOpts,
+ OptionTypeInfo::Struct(
+ kOptNameBMCompOpts, &compression_options_type_info,
+ offsetof(struct MutableCFOptions, bottommost_compression_opts),
+ OptionVerificationType::kNormal,
+ (OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
+ [](const ConfigOptions& opts, const std::string& name,
+ const std::string& value, void* addr) {
+ // This is to handle backward compatibility, where
+ // compression_options was a ":" separated list.
+ if (name == kOptNameBMCompOpts &&
+ value.find("=") == std::string::npos) {
+ auto* compression = static_cast<CompressionOptions*>(addr);
+ return ParseCompressionOptions(value, name, *compression);
+ } else {
+ return OptionTypeInfo::ParseStruct(
+ opts, kOptNameBMCompOpts, &compression_options_type_info,
+ name, value, addr);
+ }
+ })},
+ // End special case properties
+};
+
+static std::unordered_map<std::string, OptionTypeInfo>
+ cf_immutable_options_type_info = {
+ /* not yet supported
+ CompressionOptions compression_opts;
+ TablePropertiesCollectorFactories table_properties_collector_factories;
+ using TablePropertiesCollectorFactories =
+ std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>;
+ UpdateStatus (*inplace_callback)(char* existing_value,
+ uint34_t* existing_value_size,
+ Slice delta_value,
+ std::string* merged_value);
+ std::vector<DbPath> cf_paths;
+ */
+ {"compaction_measure_io_stats",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"purge_redundant_kvs_while_flush",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"inplace_update_support",
+ {offsetof(struct ImmutableCFOptions, inplace_update_support),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"level_compaction_dynamic_level_bytes",
+ {offsetof(struct ImmutableCFOptions,
+ level_compaction_dynamic_level_bytes),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"level_compaction_dynamic_file_size",
+ {offsetof(struct ImmutableCFOptions,
+ level_compaction_dynamic_file_size),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"optimize_filters_for_hits",
+ {offsetof(struct ImmutableCFOptions, optimize_filters_for_hits),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"force_consistency_checks",
+ {offsetof(struct ImmutableCFOptions, force_consistency_checks),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"preclude_last_level_data_seconds",
+ {offsetof(struct ImmutableCFOptions, preclude_last_level_data_seconds),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"preserve_internal_time_seconds",
+ {offsetof(struct ImmutableCFOptions, preserve_internal_time_seconds),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ // Need to keep this around to be able to read old OPTIONS files.
+ {"max_mem_compaction_level",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"max_write_buffer_number_to_maintain",
+ {offsetof(struct ImmutableCFOptions,
+ max_write_buffer_number_to_maintain),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone, 0}},
+ {"max_write_buffer_size_to_maintain",
+ {offsetof(struct ImmutableCFOptions,
+ max_write_buffer_size_to_maintain),
+ OptionType::kInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"min_write_buffer_number_to_merge",
+ {offsetof(struct ImmutableCFOptions, min_write_buffer_number_to_merge),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone, 0}},
+ {"num_levels",
+ {offsetof(struct ImmutableCFOptions, num_levels), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"bloom_locality",
+ {offsetof(struct ImmutableCFOptions, bloom_locality),
+ OptionType::kUInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"rate_limit_delay_max_milliseconds",
+ {0, OptionType::kUInt, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"comparator",
+ OptionTypeInfo::AsCustomRawPtr<const Comparator>(
+ offsetof(struct ImmutableCFOptions, user_comparator),
+ OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose)
+ .SetSerializeFunc(
+ // Serializes a Comparator
+ [](const ConfigOptions& opts, const std::string&,
+ const void* addr, std::string* value) {
+ // it's a const pointer of const Comparator*
+ const auto* ptr =
+ static_cast<const Comparator* const*>(addr);
+ // Since the user-specified comparator will be wrapped by
+ // InternalKeyComparator, we should persist the
+ // user-specified one instead of InternalKeyComparator.
+ if (*ptr == nullptr) {
+ *value = kNullptrString;
+ } else if (opts.mutable_options_only) {
+ *value = "";
+ } else {
+ const Comparator* root_comp = (*ptr)->GetRootComparator();
+ if (root_comp == nullptr) {
+ root_comp = (*ptr);
+ }
+ *value = root_comp->ToString(opts);
+ }
+ return Status::OK();
+ })},
+ {"memtable_insert_with_hint_prefix_extractor",
+ OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
+ offsetof(struct ImmutableCFOptions,
+ memtable_insert_with_hint_prefix_extractor),
+ OptionVerificationType::kByNameAllowNull, OptionTypeFlags::kNone)},
+ {"memtable_factory",
+ {offsetof(struct ImmutableCFOptions, memtable_factory),
+ OptionType::kCustomizable, OptionVerificationType::kByName,
+ OptionTypeFlags::kShared,
+ [](const ConfigOptions& opts, const std::string&,
+ const std::string& value, void* addr) {
+ std::unique_ptr<MemTableRepFactory> factory;
+ auto* shared =
+ static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
+ Status s =
+ MemTableRepFactory::CreateFromString(opts, value, shared);
+ return s;
+ }}},
+ {"memtable",
+ {offsetof(struct ImmutableCFOptions, memtable_factory),
+ OptionType::kCustomizable, OptionVerificationType::kAlias,
+ OptionTypeFlags::kShared,
+ [](const ConfigOptions& opts, const std::string&,
+ const std::string& value, void* addr) {
+ std::unique_ptr<MemTableRepFactory> factory;
+ auto* shared =
+ static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
+ Status s =
+ MemTableRepFactory::CreateFromString(opts, value, shared);
+ return s;
+ }}},
+ {"table_factory",
+ OptionTypeInfo::AsCustomSharedPtr<TableFactory>(
+ offsetof(struct ImmutableCFOptions, table_factory),
+ OptionVerificationType::kByName,
+ (OptionTypeFlags::kCompareLoose |
+ OptionTypeFlags::kStringNameOnly |
+ OptionTypeFlags::kDontPrepare))},
+ {"block_based_table_factory",
+ {offsetof(struct ImmutableCFOptions, table_factory),
+ OptionType::kCustomizable, OptionVerificationType::kAlias,
+ OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
+ // Parses the input value and creates a BlockBasedTableFactory
+ [](const ConfigOptions& opts, const std::string& name,
+ const std::string& value, void* addr) {
+ BlockBasedTableOptions* old_opts = nullptr;
+ auto table_factory =
+ static_cast<std::shared_ptr<TableFactory>*>(addr);
+ if (table_factory->get() != nullptr) {
+ old_opts =
+ table_factory->get()->GetOptions<BlockBasedTableOptions>();
+ }
+ if (name == "block_based_table_factory") {
+ std::unique_ptr<TableFactory> new_factory;
+ if (old_opts != nullptr) {
+ new_factory.reset(NewBlockBasedTableFactory(*old_opts));
+ } else {
+ new_factory.reset(NewBlockBasedTableFactory());
+ }
+ Status s = new_factory->ConfigureFromString(opts, value);
+ if (s.ok()) {
+ table_factory->reset(new_factory.release());
+ }
+ return s;
+ } else if (old_opts != nullptr) {
+ return table_factory->get()->ConfigureOption(opts, name, value);
+ } else {
+ return Status::NotFound("Mismatched table option: ", name);
+ }
+ }}},
+ {"plain_table_factory",
+ {offsetof(struct ImmutableCFOptions, table_factory),
+ OptionType::kCustomizable, OptionVerificationType::kAlias,
+ OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
+ // Parses the input value and creates a PlainTableFactory
+ [](const ConfigOptions& opts, const std::string& name,
+ const std::string& value, void* addr) {
+ PlainTableOptions* old_opts = nullptr;
+ auto table_factory =
+ static_cast<std::shared_ptr<TableFactory>*>(addr);
+ if (table_factory->get() != nullptr) {
+ old_opts = table_factory->get()->GetOptions<PlainTableOptions>();
+ }
+ if (name == "plain_table_factory") {
+ std::unique_ptr<TableFactory> new_factory;
+ if (old_opts != nullptr) {
+ new_factory.reset(NewPlainTableFactory(*old_opts));
+ } else {
+ new_factory.reset(NewPlainTableFactory());
+ }
+ Status s = new_factory->ConfigureFromString(opts, value);
+ if (s.ok()) {
+ table_factory->reset(new_factory.release());
+ }
+ return s;
+ } else if (old_opts != nullptr) {
+ return table_factory->get()->ConfigureOption(opts, name, value);
+ } else {
+ return Status::NotFound("Mismatched table option: ", name);
+ }
+ }}},
+ {"table_properties_collectors",
+ OptionTypeInfo::Vector<
+ std::shared_ptr<TablePropertiesCollectorFactory>>(
+ offsetof(struct ImmutableCFOptions,
+ table_properties_collector_factories),
+ OptionVerificationType::kByName, OptionTypeFlags::kNone,
+ OptionTypeInfo::AsCustomSharedPtr<TablePropertiesCollectorFactory>(
+ 0, OptionVerificationType::kByName, OptionTypeFlags::kNone))},
+ {"compaction_filter",
+ OptionTypeInfo::AsCustomRawPtr<const CompactionFilter>(
+ offsetof(struct ImmutableCFOptions, compaction_filter),
+ OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
+ {"compaction_filter_factory",
+ OptionTypeInfo::AsCustomSharedPtr<CompactionFilterFactory>(
+ offsetof(struct ImmutableCFOptions, compaction_filter_factory),
+ OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
+ {"merge_operator",
+ OptionTypeInfo::AsCustomSharedPtr<MergeOperator>(
+ offsetof(struct ImmutableCFOptions, merge_operator),
+ OptionVerificationType::kByNameAllowFromNull,
+ OptionTypeFlags::kCompareLoose | OptionTypeFlags::kAllowNull)},
+ {"compaction_style",
+ {offsetof(struct ImmutableCFOptions, compaction_style),
+ OptionType::kCompactionStyle, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"compaction_pri",
+ {offsetof(struct ImmutableCFOptions, compaction_pri),
+ OptionType::kCompactionPri, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"sst_partitioner_factory",
+ OptionTypeInfo::AsCustomSharedPtr<SstPartitionerFactory>(
+ offsetof(struct ImmutableCFOptions, sst_partitioner_factory),
+ OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
+ {"blob_cache",
+ {offsetof(struct ImmutableCFOptions, blob_cache), OptionType::kUnknown,
+ OptionVerificationType::kNormal,
+ (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize),
+ // Parses the input value as a Cache
+ [](const ConfigOptions& opts, const std::string&,
+ const std::string& value, void* addr) {
+ auto* cache = static_cast<std::shared_ptr<Cache>*>(addr);
+ return Cache::CreateFromString(opts, value, cache);
+ }}},
+};
+
+const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions";
+
+class ConfigurableMutableCFOptions : public Configurable {
+ public:
+ explicit ConfigurableMutableCFOptions(const MutableCFOptions& mcf) {
+ mutable_ = mcf;
+ RegisterOptions(&mutable_, &cf_mutable_options_type_info);
+ }
+
+ protected:
+ MutableCFOptions mutable_;
+};
+
+class ConfigurableCFOptions : public ConfigurableMutableCFOptions {
+ public:
+ ConfigurableCFOptions(const ColumnFamilyOptions& opts,
+ const std::unordered_map<std::string, std::string>* map)
+ : ConfigurableMutableCFOptions(MutableCFOptions(opts)),
+ immutable_(opts),
+ cf_options_(opts),
+ opt_map_(map) {
+ RegisterOptions(&immutable_, &cf_immutable_options_type_info);
+ }
+
+ protected:
+ Status ConfigureOptions(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ std::unordered_map<std::string, std::string>* unused) override {
+ Status s = Configurable::ConfigureOptions(config_options, opts_map, unused);
+ if (s.ok()) {
+ UpdateColumnFamilyOptions(mutable_, &cf_options_);
+ UpdateColumnFamilyOptions(immutable_, &cf_options_);
+ s = PrepareOptions(config_options);
+ }
+ return s;
+ }
+
+ virtual const void* GetOptionsPtr(const std::string& name) const override {
+ if (name == OptionsHelper::kCFOptionsName) {
+ return &cf_options_;
+ } else {
+ return ConfigurableMutableCFOptions::GetOptionsPtr(name);
+ }
+ }
+
+ bool OptionsAreEqual(const ConfigOptions& config_options,
+ const OptionTypeInfo& opt_info,
+ const std::string& opt_name, const void* const this_ptr,
+ const void* const that_ptr,
+ std::string* mismatch) const override {
+ bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr,
+ that_ptr, mismatch);
+ if (!equals && opt_info.IsByName()) {
+ if (opt_map_ == nullptr) {
+ equals = true;
+ } else {
+ const auto& iter = opt_map_->find(opt_name);
+ if (iter == opt_map_->end()) {
+ equals = true;
+ } else {
+ equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr,
+ iter->second);
+ }
+ }
+ if (equals) { // False alarm, clear mismatch
+ *mismatch = "";
+ }
+ }
+ if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) {
+ const auto* this_config = opt_info.AsRawPointer<Configurable>(this_ptr);
+ if (this_config == nullptr) {
+ const auto& iter = opt_map_->find(opt_name);
+ // If the name exists in the map and is not empty/null,
+ // then the this_config should be set.
+ if (iter != opt_map_->end() && !iter->second.empty() &&
+ iter->second != kNullptrString) {
+ *mismatch = opt_name;
+ equals = false;
+ }
+ }
+ }
+ return equals;
+ }
+
+ private:
+ ImmutableCFOptions immutable_;
+ ColumnFamilyOptions cf_options_;
+ const std::unordered_map<std::string, std::string>* opt_map_;
+};
+
+std::unique_ptr<Configurable> CFOptionsAsConfigurable(
+ const MutableCFOptions& opts) {
+ std::unique_ptr<Configurable> ptr(new ConfigurableMutableCFOptions(opts));
+ return ptr;
+}
+std::unique_ptr<Configurable> CFOptionsAsConfigurable(
+ const ColumnFamilyOptions& opts,
+ const std::unordered_map<std::string, std::string>* opt_map) {
+ std::unique_ptr<Configurable> ptr(new ConfigurableCFOptions(opts, opt_map));
+ return ptr;
+}
+#endif // ROCKSDB_LITE
+
+ImmutableCFOptions::ImmutableCFOptions() : ImmutableCFOptions(Options()) {}
+
+ImmutableCFOptions::ImmutableCFOptions(const ColumnFamilyOptions& cf_options)
+ : compaction_style(cf_options.compaction_style),
+ compaction_pri(cf_options.compaction_pri),
+ user_comparator(cf_options.comparator),
+ internal_comparator(InternalKeyComparator(cf_options.comparator)),
+ merge_operator(cf_options.merge_operator),
+ compaction_filter(cf_options.compaction_filter),
+ compaction_filter_factory(cf_options.compaction_filter_factory),
+ min_write_buffer_number_to_merge(
+ cf_options.min_write_buffer_number_to_merge),
+ max_write_buffer_number_to_maintain(
+ cf_options.max_write_buffer_number_to_maintain),
+ max_write_buffer_size_to_maintain(
+ cf_options.max_write_buffer_size_to_maintain),
+ inplace_update_support(cf_options.inplace_update_support),
+ inplace_callback(cf_options.inplace_callback),
+ memtable_factory(cf_options.memtable_factory),
+ table_factory(cf_options.table_factory),
+ table_properties_collector_factories(
+ cf_options.table_properties_collector_factories),
+ bloom_locality(cf_options.bloom_locality),
+ level_compaction_dynamic_level_bytes(
+ cf_options.level_compaction_dynamic_level_bytes),
+ level_compaction_dynamic_file_size(
+ cf_options.level_compaction_dynamic_file_size),
+ num_levels(cf_options.num_levels),
+ optimize_filters_for_hits(cf_options.optimize_filters_for_hits),
+ force_consistency_checks(cf_options.force_consistency_checks),
+ preclude_last_level_data_seconds(
+ cf_options.preclude_last_level_data_seconds),
+ preserve_internal_time_seconds(cf_options.preserve_internal_time_seconds),
+ memtable_insert_with_hint_prefix_extractor(
+ cf_options.memtable_insert_with_hint_prefix_extractor),
+ cf_paths(cf_options.cf_paths),
+ compaction_thread_limiter(cf_options.compaction_thread_limiter),
+ sst_partitioner_factory(cf_options.sst_partitioner_factory),
+ blob_cache(cf_options.blob_cache) {}
+
+ImmutableOptions::ImmutableOptions() : ImmutableOptions(Options()) {}
+
+ImmutableOptions::ImmutableOptions(const Options& options)
+ : ImmutableOptions(options, options) {}
+
+ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
+ const ColumnFamilyOptions& cf_options)
+ : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
+
+ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
+ const ImmutableCFOptions& cf_options)
+ : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
+
+ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
+ const ColumnFamilyOptions& cf_options)
+ : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
+
+ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
+ const ImmutableCFOptions& cf_options)
+ : ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
+
+// Multiple two operands. If they overflow, return op1.
+uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
+ if (op1 == 0 || op2 <= 0) {
+ return 0;
+ }
+ if (std::numeric_limits<uint64_t>::max() / op1 < op2) {
+ return op1;
+ }
+ return static_cast<uint64_t>(op1 * op2);
+}
+
+// when level_compaction_dynamic_level_bytes is true and leveled compaction
+// is used, the base level is not always L1, so precomupted max_file_size can
+// no longer be used. Recompute file_size_for_level from base level.
+uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
+ int level, CompactionStyle compaction_style, int base_level,
+ bool level_compaction_dynamic_level_bytes) {
+ if (!level_compaction_dynamic_level_bytes || level < base_level ||
+ compaction_style != kCompactionStyleLevel) {
+ assert(level >= 0);
+ assert(level < (int)cf_options.max_file_size.size());
+ return cf_options.max_file_size[level];
+ } else {
+ assert(level >= 0 && base_level >= 0);
+ assert(level - base_level < (int)cf_options.max_file_size.size());
+ return cf_options.max_file_size[level - base_level];
+ }
+}
+
+size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options) {
+ // We do not want to pin meta-blocks that almost certainly came from intra-L0
+ // or a former larger `write_buffer_size` value to avoid surprising users with
+ // pinned memory usage. We use a factor of 1.5 to account for overhead
+ // introduced during flush in most cases.
+ if (std::numeric_limits<size_t>::max() / 3 <
+ cf_options.write_buffer_size / 2) {
+ return std::numeric_limits<size_t>::max();
+ }
+ return cf_options.write_buffer_size / 2 * 3;
+}
+
+void MutableCFOptions::RefreshDerivedOptions(int num_levels,
+ CompactionStyle compaction_style) {
+ max_file_size.resize(num_levels);
+ for (int i = 0; i < num_levels; ++i) {
+ if (i == 0 && compaction_style == kCompactionStyleUniversal) {
+ max_file_size[i] = ULLONG_MAX;
+ } else if (i > 1) {
+ max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1],
+ target_file_size_multiplier);
+ } else {
+ max_file_size[i] = target_file_size_base;
+ }
+ }
+}
+
+void MutableCFOptions::Dump(Logger* log) const {
+ // Memtable related options
+ ROCKS_LOG_INFO(log,
+ " write_buffer_size: %" ROCKSDB_PRIszt,
+ write_buffer_size);
+ ROCKS_LOG_INFO(log, " max_write_buffer_number: %d",
+ max_write_buffer_number);
+ ROCKS_LOG_INFO(log,
+ " arena_block_size: %" ROCKSDB_PRIszt,
+ arena_block_size);
+ ROCKS_LOG_INFO(log, " memtable_prefix_bloom_ratio: %f",
+ memtable_prefix_bloom_size_ratio);
+ ROCKS_LOG_INFO(log, " memtable_whole_key_filtering: %d",
+ memtable_whole_key_filtering);
+ ROCKS_LOG_INFO(log,
+ " memtable_huge_page_size: %" ROCKSDB_PRIszt,
+ memtable_huge_page_size);
+ ROCKS_LOG_INFO(log,
+ " max_successive_merges: %" ROCKSDB_PRIszt,
+ max_successive_merges);
+ ROCKS_LOG_INFO(log,
+ " inplace_update_num_locks: %" ROCKSDB_PRIszt,
+ inplace_update_num_locks);
+ ROCKS_LOG_INFO(log, " prefix_extractor: %s",
+ prefix_extractor == nullptr
+ ? "nullptr"
+ : prefix_extractor->GetId().c_str());
+ ROCKS_LOG_INFO(log, " disable_auto_compactions: %d",
+ disable_auto_compactions);
+ ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
+ soft_pending_compaction_bytes_limit);
+ ROCKS_LOG_INFO(log, " hard_pending_compaction_bytes_limit: %" PRIu64,
+ hard_pending_compaction_bytes_limit);
+ ROCKS_LOG_INFO(log, " level0_file_num_compaction_trigger: %d",
+ level0_file_num_compaction_trigger);
+ ROCKS_LOG_INFO(log, " level0_slowdown_writes_trigger: %d",
+ level0_slowdown_writes_trigger);
+ ROCKS_LOG_INFO(log, " level0_stop_writes_trigger: %d",
+ level0_stop_writes_trigger);
+ ROCKS_LOG_INFO(log, " max_compaction_bytes: %" PRIu64,
+ max_compaction_bytes);
+ ROCKS_LOG_INFO(log, " ignore_max_compaction_bytes_for_input: %s",
+ ignore_max_compaction_bytes_for_input ? "true" : "false");
+ ROCKS_LOG_INFO(log, " target_file_size_base: %" PRIu64,
+ target_file_size_base);
+ ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d",
+ target_file_size_multiplier);
+ ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64,
+ max_bytes_for_level_base);
+ ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f",
+ max_bytes_for_level_multiplier);
+ ROCKS_LOG_INFO(log, " ttl: %" PRIu64,
+ ttl);
+ ROCKS_LOG_INFO(log, " periodic_compaction_seconds: %" PRIu64,
+ periodic_compaction_seconds);
+ std::string result;
+ char buf[10];
+ for (const auto m : max_bytes_for_level_multiplier_additional) {
+ snprintf(buf, sizeof(buf), "%d, ", m);
+ result += buf;
+ }
+ if (result.size() >= 2) {
+ result.resize(result.size() - 2);
+ } else {
+ result = "";
+ }
+
+ ROCKS_LOG_INFO(log, "max_bytes_for_level_multiplier_additional: %s",
+ result.c_str());
+ ROCKS_LOG_INFO(log, " max_sequential_skip_in_iterations: %" PRIu64,
+ max_sequential_skip_in_iterations);
+ ROCKS_LOG_INFO(log, " check_flush_compaction_key_order: %d",
+ check_flush_compaction_key_order);
+ ROCKS_LOG_INFO(log, " paranoid_file_checks: %d",
+ paranoid_file_checks);
+ ROCKS_LOG_INFO(log, " report_bg_io_stats: %d",
+ report_bg_io_stats);
+ ROCKS_LOG_INFO(log, " compression: %d",
+ static_cast<int>(compression));
+ ROCKS_LOG_INFO(log,
+ " experimental_mempurge_threshold: %f",
+ experimental_mempurge_threshold);
+
+ // Universal Compaction Options
+ ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d",
+ compaction_options_universal.size_ratio);
+ ROCKS_LOG_INFO(log, "compaction_options_universal.min_merge_width : %d",
+ compaction_options_universal.min_merge_width);
+ ROCKS_LOG_INFO(log, "compaction_options_universal.max_merge_width : %d",
+ compaction_options_universal.max_merge_width);
+ ROCKS_LOG_INFO(
+ log, "compaction_options_universal.max_size_amplification_percent : %d",
+ compaction_options_universal.max_size_amplification_percent);
+ ROCKS_LOG_INFO(log,
+ "compaction_options_universal.compression_size_percent : %d",
+ compaction_options_universal.compression_size_percent);
+ ROCKS_LOG_INFO(log, "compaction_options_universal.stop_style : %d",
+ compaction_options_universal.stop_style);
+ ROCKS_LOG_INFO(
+ log, "compaction_options_universal.allow_trivial_move : %d",
+ static_cast<int>(compaction_options_universal.allow_trivial_move));
+ ROCKS_LOG_INFO(log, "compaction_options_universal.incremental : %d",
+ static_cast<int>(compaction_options_universal.incremental));
+
+ // FIFO Compaction Options
+ ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64,
+ compaction_options_fifo.max_table_files_size);
+ ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d",
+ compaction_options_fifo.allow_compaction);
+
+ // Blob file related options
+ ROCKS_LOG_INFO(log, " enable_blob_files: %s",
+ enable_blob_files ? "true" : "false");
+ ROCKS_LOG_INFO(log, " min_blob_size: %" PRIu64,
+ min_blob_size);
+ ROCKS_LOG_INFO(log, " blob_file_size: %" PRIu64,
+ blob_file_size);
+ ROCKS_LOG_INFO(log, " blob_compression_type: %s",
+ CompressionTypeToString(blob_compression_type).c_str());
+ ROCKS_LOG_INFO(log, " enable_blob_garbage_collection: %s",
+ enable_blob_garbage_collection ? "true" : "false");
+ ROCKS_LOG_INFO(log, " blob_garbage_collection_age_cutoff: %f",
+ blob_garbage_collection_age_cutoff);
+ ROCKS_LOG_INFO(log, " blob_garbage_collection_force_threshold: %f",
+ blob_garbage_collection_force_threshold);
+ ROCKS_LOG_INFO(log, " blob_compaction_readahead_size: %" PRIu64,
+ blob_compaction_readahead_size);
+ ROCKS_LOG_INFO(log, " blob_file_starting_level: %d",
+ blob_file_starting_level);
+ ROCKS_LOG_INFO(log, " prepopulate_blob_cache: %s",
+ prepopulate_blob_cache == PrepopulateBlobCache::kFlushOnly
+ ? "flush only"
+ : "disable");
+ ROCKS_LOG_INFO(log, " last_level_temperature: %d",
+ static_cast<int>(last_level_temperature));
+}
+
+MutableCFOptions::MutableCFOptions(const Options& options)
+ : MutableCFOptions(ColumnFamilyOptions(options)) {}
+
+#ifndef ROCKSDB_LITE
+Status GetMutableOptionsFromStrings(
+ const MutableCFOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ Logger* /*info_log*/, MutableCFOptions* new_options) {
+ assert(new_options);
+ *new_options = base_options;
+ ConfigOptions config_options;
+ Status s = OptionTypeInfo::ParseType(
+ config_options, options_map, cf_mutable_options_type_info, new_options);
+ if (!s.ok()) {
+ *new_options = base_options;
+ }
+ return s;
+}
+
+Status GetStringFromMutableCFOptions(const ConfigOptions& config_options,
+ const MutableCFOptions& mutable_opts,
+ std::string* opt_string) {
+ assert(opt_string);
+ opt_string->clear();
+ return OptionTypeInfo::SerializeType(
+ config_options, cf_mutable_options_type_info, &mutable_opts, opt_string);
+}
+#endif // ROCKSDB_LITE
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/cf_options.h b/src/rocksdb/options/cf_options.h
new file mode 100644
index 000000000..050618eda
--- /dev/null
+++ b/src/rocksdb/options/cf_options.h
@@ -0,0 +1,344 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "db/dbformat.h"
+#include "options/db_options.h"
+#include "rocksdb/options.h"
+#include "util/compression.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// ImmutableCFOptions is a data struct used by RocksDB internal. It contains a
+// subset of Options that should not be changed during the entire lifetime
+// of DB. Raw pointers defined in this struct do not have ownership to the data
+// they point to. Options contains std::shared_ptr to these data.
+struct ImmutableCFOptions {
+ public:
+ static const char* kName() { return "ImmutableCFOptions"; }
+ explicit ImmutableCFOptions();
+ explicit ImmutableCFOptions(const ColumnFamilyOptions& cf_options);
+
+ CompactionStyle compaction_style;
+
+ CompactionPri compaction_pri;
+
+ const Comparator* user_comparator;
+ InternalKeyComparator internal_comparator; // Only in Immutable
+
+ std::shared_ptr<MergeOperator> merge_operator;
+
+ const CompactionFilter* compaction_filter;
+
+ std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
+
+ int min_write_buffer_number_to_merge;
+
+ int max_write_buffer_number_to_maintain;
+
+ int64_t max_write_buffer_size_to_maintain;
+
+ bool inplace_update_support;
+
+ UpdateStatus (*inplace_callback)(char* existing_value,
+ uint32_t* existing_value_size,
+ Slice delta_value,
+ std::string* merged_value);
+
+ std::shared_ptr<MemTableRepFactory> memtable_factory;
+
+ std::shared_ptr<TableFactory> table_factory;
+
+ Options::TablePropertiesCollectorFactories
+ table_properties_collector_factories;
+
+ // This options is required by PlainTableReader. May need to move it
+ // to PlainTableOptions just like bloom_bits_per_key
+ uint32_t bloom_locality;
+
+ bool level_compaction_dynamic_level_bytes;
+
+ bool level_compaction_dynamic_file_size;
+
+ int num_levels;
+
+ bool optimize_filters_for_hits;
+
+ bool force_consistency_checks;
+
+ uint64_t preclude_last_level_data_seconds;
+
+ uint64_t preserve_internal_time_seconds;
+
+ std::shared_ptr<const SliceTransform>
+ memtable_insert_with_hint_prefix_extractor;
+
+ std::vector<DbPath> cf_paths;
+
+ std::shared_ptr<ConcurrentTaskLimiter> compaction_thread_limiter;
+
+ std::shared_ptr<SstPartitionerFactory> sst_partitioner_factory;
+
+ std::shared_ptr<Cache> blob_cache;
+};
+
+struct ImmutableOptions : public ImmutableDBOptions, public ImmutableCFOptions {
+ explicit ImmutableOptions();
+ explicit ImmutableOptions(const Options& options);
+
+ ImmutableOptions(const DBOptions& db_options,
+ const ColumnFamilyOptions& cf_options);
+
+ ImmutableOptions(const ImmutableDBOptions& db_options,
+ const ImmutableCFOptions& cf_options);
+
+ ImmutableOptions(const DBOptions& db_options,
+ const ImmutableCFOptions& cf_options);
+
+ ImmutableOptions(const ImmutableDBOptions& db_options,
+ const ColumnFamilyOptions& cf_options);
+};
+
+struct MutableCFOptions {
+ static const char* kName() { return "MutableCFOptions"; }
+ explicit MutableCFOptions(const ColumnFamilyOptions& options)
+ : write_buffer_size(options.write_buffer_size),
+ max_write_buffer_number(options.max_write_buffer_number),
+ arena_block_size(options.arena_block_size),
+ memtable_prefix_bloom_size_ratio(
+ options.memtable_prefix_bloom_size_ratio),
+ memtable_whole_key_filtering(options.memtable_whole_key_filtering),
+ memtable_huge_page_size(options.memtable_huge_page_size),
+ max_successive_merges(options.max_successive_merges),
+ inplace_update_num_locks(options.inplace_update_num_locks),
+ prefix_extractor(options.prefix_extractor),
+ experimental_mempurge_threshold(
+ options.experimental_mempurge_threshold),
+ disable_auto_compactions(options.disable_auto_compactions),
+ soft_pending_compaction_bytes_limit(
+ options.soft_pending_compaction_bytes_limit),
+ hard_pending_compaction_bytes_limit(
+ options.hard_pending_compaction_bytes_limit),
+ level0_file_num_compaction_trigger(
+ options.level0_file_num_compaction_trigger),
+ level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
+ level0_stop_writes_trigger(options.level0_stop_writes_trigger),
+ max_compaction_bytes(options.max_compaction_bytes),
+ ignore_max_compaction_bytes_for_input(
+ options.ignore_max_compaction_bytes_for_input),
+ target_file_size_base(options.target_file_size_base),
+ target_file_size_multiplier(options.target_file_size_multiplier),
+ max_bytes_for_level_base(options.max_bytes_for_level_base),
+ max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
+ ttl(options.ttl),
+ periodic_compaction_seconds(options.periodic_compaction_seconds),
+ max_bytes_for_level_multiplier_additional(
+ options.max_bytes_for_level_multiplier_additional),
+ compaction_options_fifo(options.compaction_options_fifo),
+ compaction_options_universal(options.compaction_options_universal),
+ enable_blob_files(options.enable_blob_files),
+ min_blob_size(options.min_blob_size),
+ blob_file_size(options.blob_file_size),
+ blob_compression_type(options.blob_compression_type),
+ enable_blob_garbage_collection(options.enable_blob_garbage_collection),
+ blob_garbage_collection_age_cutoff(
+ options.blob_garbage_collection_age_cutoff),
+ blob_garbage_collection_force_threshold(
+ options.blob_garbage_collection_force_threshold),
+ blob_compaction_readahead_size(options.blob_compaction_readahead_size),
+ blob_file_starting_level(options.blob_file_starting_level),
+ prepopulate_blob_cache(options.prepopulate_blob_cache),
+ max_sequential_skip_in_iterations(
+ options.max_sequential_skip_in_iterations),
+ check_flush_compaction_key_order(
+ options.check_flush_compaction_key_order),
+ paranoid_file_checks(options.paranoid_file_checks),
+ report_bg_io_stats(options.report_bg_io_stats),
+ compression(options.compression),
+ bottommost_compression(options.bottommost_compression),
+ compression_opts(options.compression_opts),
+ bottommost_compression_opts(options.bottommost_compression_opts),
+ last_level_temperature(options.last_level_temperature ==
+ Temperature::kUnknown
+ ? options.bottommost_temperature
+ : options.last_level_temperature),
+ memtable_protection_bytes_per_key(
+ options.memtable_protection_bytes_per_key),
+ sample_for_compression(
+ options.sample_for_compression), // TODO: is 0 fine here?
+ compression_per_level(options.compression_per_level) {
+ RefreshDerivedOptions(options.num_levels, options.compaction_style);
+ }
+
+ MutableCFOptions()
+ : write_buffer_size(0),
+ max_write_buffer_number(0),
+ arena_block_size(0),
+ memtable_prefix_bloom_size_ratio(0),
+ memtable_whole_key_filtering(false),
+ memtable_huge_page_size(0),
+ max_successive_merges(0),
+ inplace_update_num_locks(0),
+ prefix_extractor(nullptr),
+ experimental_mempurge_threshold(0.0),
+ disable_auto_compactions(false),
+ soft_pending_compaction_bytes_limit(0),
+ hard_pending_compaction_bytes_limit(0),
+ level0_file_num_compaction_trigger(0),
+ level0_slowdown_writes_trigger(0),
+ level0_stop_writes_trigger(0),
+ max_compaction_bytes(0),
+ ignore_max_compaction_bytes_for_input(true),
+ target_file_size_base(0),
+ target_file_size_multiplier(0),
+ max_bytes_for_level_base(0),
+ max_bytes_for_level_multiplier(0),
+ ttl(0),
+ periodic_compaction_seconds(0),
+ compaction_options_fifo(),
+ enable_blob_files(false),
+ min_blob_size(0),
+ blob_file_size(0),
+ blob_compression_type(kNoCompression),
+ enable_blob_garbage_collection(false),
+ blob_garbage_collection_age_cutoff(0.0),
+ blob_garbage_collection_force_threshold(0.0),
+ blob_compaction_readahead_size(0),
+ blob_file_starting_level(0),
+ prepopulate_blob_cache(PrepopulateBlobCache::kDisable),
+ max_sequential_skip_in_iterations(0),
+ check_flush_compaction_key_order(true),
+ paranoid_file_checks(false),
+ report_bg_io_stats(false),
+ compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
+ bottommost_compression(kDisableCompressionOption),
+ last_level_temperature(Temperature::kUnknown),
+ memtable_protection_bytes_per_key(0),
+ sample_for_compression(0) {}
+
+ explicit MutableCFOptions(const Options& options);
+
+ // Must be called after any change to MutableCFOptions
+ void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style);
+
+ void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) {
+ RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style);
+ }
+
+ int MaxBytesMultiplerAdditional(int level) const {
+ if (level >=
+ static_cast<int>(max_bytes_for_level_multiplier_additional.size())) {
+ return 1;
+ }
+ return max_bytes_for_level_multiplier_additional[level];
+ }
+
+ void Dump(Logger* log) const;
+
+ // Memtable related options
+ size_t write_buffer_size;
+ int max_write_buffer_number;
+ size_t arena_block_size;
+ double memtable_prefix_bloom_size_ratio;
+ bool memtable_whole_key_filtering;
+ size_t memtable_huge_page_size;
+ size_t max_successive_merges;
+ size_t inplace_update_num_locks;
+ std::shared_ptr<const SliceTransform> prefix_extractor;
+ // [experimental]
+ // Used to activate or deactive the Mempurge feature (memtable garbage
+ // collection). (deactivated by default). At every flush, the total useful
+ // payload (total entries minus garbage entries) is estimated as a ratio
+ // [useful payload bytes]/[size of a memtable (in bytes)]. This ratio is then
+ // compared to this `threshold` value:
+ // - if ratio<threshold: the flush is replaced by a mempurge operation
+ // - else: a regular flush operation takes place.
+ // Threshold values:
+ // 0.0: mempurge deactivated (default).
+ // 1.0: recommended threshold value.
+ // >1.0 : aggressive mempurge.
+ // 0 < threshold < 1.0: mempurge triggered only for very low useful payload
+ // ratios.
+ // [experimental]
+ double experimental_mempurge_threshold;
+
+ // Compaction related options
+ bool disable_auto_compactions;
+ uint64_t soft_pending_compaction_bytes_limit;
+ uint64_t hard_pending_compaction_bytes_limit;
+ int level0_file_num_compaction_trigger;
+ int level0_slowdown_writes_trigger;
+ int level0_stop_writes_trigger;
+ uint64_t max_compaction_bytes;
+ bool ignore_max_compaction_bytes_for_input;
+ uint64_t target_file_size_base;
+ int target_file_size_multiplier;
+ uint64_t max_bytes_for_level_base;
+ double max_bytes_for_level_multiplier;
+ uint64_t ttl;
+ uint64_t periodic_compaction_seconds;
+ std::vector<int> max_bytes_for_level_multiplier_additional;
+ CompactionOptionsFIFO compaction_options_fifo;
+ CompactionOptionsUniversal compaction_options_universal;
+
+ // Blob file related options
+ bool enable_blob_files;
+ uint64_t min_blob_size;
+ uint64_t blob_file_size;
+ CompressionType blob_compression_type;
+ bool enable_blob_garbage_collection;
+ double blob_garbage_collection_age_cutoff;
+ double blob_garbage_collection_force_threshold;
+ uint64_t blob_compaction_readahead_size;
+ int blob_file_starting_level;
+ PrepopulateBlobCache prepopulate_blob_cache;
+
+ // Misc options
+ uint64_t max_sequential_skip_in_iterations;
+ bool check_flush_compaction_key_order;
+ bool paranoid_file_checks;
+ bool report_bg_io_stats;
+ CompressionType compression;
+ CompressionType bottommost_compression;
+ CompressionOptions compression_opts;
+ CompressionOptions bottommost_compression_opts;
+ Temperature last_level_temperature;
+ uint32_t memtable_protection_bytes_per_key;
+
+ uint64_t sample_for_compression;
+ std::vector<CompressionType> compression_per_level;
+
+ // Derived options
+ // Per-level target file size.
+ std::vector<uint64_t> max_file_size;
+};
+
+uint64_t MultiplyCheckOverflow(uint64_t op1, double op2);
+
+// Get the max file size in a given level.
+uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
+ int level, CompactionStyle compaction_style, int base_level = 1,
+ bool level_compaction_dynamic_level_bytes = false);
+
+// Get the max size of an L0 file for which we will pin its meta-blocks when
+// `pin_l0_filter_and_index_blocks_in_cache` is set.
+size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options);
+
+#ifndef ROCKSDB_LITE
+Status GetStringFromMutableCFOptions(const ConfigOptions& config_options,
+ const MutableCFOptions& mutable_opts,
+ std::string* opt_string);
+
+Status GetMutableOptionsFromStrings(
+ const MutableCFOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ Logger* info_log, MutableCFOptions* new_options);
+#endif // ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/configurable.cc b/src/rocksdb/options/configurable.cc
new file mode 100644
index 000000000..08aff10fd
--- /dev/null
+++ b/src/rocksdb/options/configurable.cc
@@ -0,0 +1,767 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "rocksdb/configurable.h"
+
+#include "logging/logging.h"
+#include "options/configurable_helper.h"
+#include "options/options_helper.h"
+#include "rocksdb/customizable.h"
+#include "rocksdb/status.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_type.h"
+#include "util/coding.h"
+#include "util/string_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+void Configurable::RegisterOptions(
+ const std::string& name, void* opt_ptr,
+ const std::unordered_map<std::string, OptionTypeInfo>* type_map) {
+ RegisteredOptions opts;
+ opts.name = name;
+#ifndef ROCKSDB_LITE
+ opts.type_map = type_map;
+#else
+ (void)type_map;
+#endif // ROCKSDB_LITE
+ opts.opt_ptr = opt_ptr;
+ options_.emplace_back(opts);
+}
+
+//*************************************************************************
+//
+// Methods for Initializing and Validating Configurable Objects
+//
+//*************************************************************************
+
+Status Configurable::PrepareOptions(const ConfigOptions& opts) {
+ // We ignore the invoke_prepare_options here intentionally,
+ // as if you are here, you must have called PrepareOptions explicitly.
+ Status status = Status::OK();
+#ifndef ROCKSDB_LITE
+ for (auto opt_iter : options_) {
+ if (opt_iter.type_map != nullptr) {
+ for (auto map_iter : *(opt_iter.type_map)) {
+ auto& opt_info = map_iter.second;
+ if (opt_info.ShouldPrepare()) {
+ status = opt_info.Prepare(opts, map_iter.first, opt_iter.opt_ptr);
+ if (!status.ok()) {
+ return status;
+ }
+ }
+ }
+ }
+ }
+#else
+ (void)opts;
+#endif // ROCKSDB_LITE
+ return status;
+}
+
+Status Configurable::ValidateOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts) const {
+ Status status;
+#ifndef ROCKSDB_LITE
+ for (auto opt_iter : options_) {
+ if (opt_iter.type_map != nullptr) {
+ for (auto map_iter : *(opt_iter.type_map)) {
+ auto& opt_info = map_iter.second;
+ if (opt_info.ShouldValidate()) {
+ status = opt_info.Validate(db_opts, cf_opts, map_iter.first,
+ opt_iter.opt_ptr);
+ if (!status.ok()) {
+ return status;
+ }
+ }
+ }
+ }
+ }
+#else
+ (void)db_opts;
+ (void)cf_opts;
+#endif // ROCKSDB_LITE
+ return status;
+}
+
+/*********************************************************************************/
+/* */
+/* Methods for Retrieving Options from Configurables */
+/* */
+/*********************************************************************************/
+
+const void* Configurable::GetOptionsPtr(const std::string& name) const {
+ for (auto o : options_) {
+ if (o.name == name) {
+ return o.opt_ptr;
+ }
+ }
+ return nullptr;
+}
+
+std::string Configurable::GetOptionName(const std::string& opt_name) const {
+ return opt_name;
+}
+
+#ifndef ROCKSDB_LITE
+const OptionTypeInfo* ConfigurableHelper::FindOption(
+ const std::vector<Configurable::RegisteredOptions>& options,
+ const std::string& short_name, std::string* opt_name, void** opt_ptr) {
+ for (auto iter : options) {
+ if (iter.type_map != nullptr) {
+ const auto opt_info =
+ OptionTypeInfo::Find(short_name, *(iter.type_map), opt_name);
+ if (opt_info != nullptr) {
+ *opt_ptr = iter.opt_ptr;
+ return opt_info;
+ }
+ }
+ }
+ return nullptr;
+}
+#endif // ROCKSDB_LITE
+
+//*************************************************************************
+//
+// Methods for Configuring Options from Strings/Name-Value Pairs/Maps
+//
+//*************************************************************************
+
+Status Configurable::ConfigureFromMap(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, std::string>& opts_map) {
+ Status s = ConfigureFromMap(config_options, opts_map, nullptr);
+ return s;
+}
+
+Status Configurable::ConfigureFromMap(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ std::unordered_map<std::string, std::string>* unused) {
+ return ConfigureOptions(config_options, opts_map, unused);
+}
+
+Status Configurable::ConfigureOptions(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ std::unordered_map<std::string, std::string>* unused) {
+ std::string curr_opts;
+ Status s;
+ if (!opts_map.empty()) {
+ // There are options in the map.
+ // Save the current configuration in curr_opts and then configure the
+ // options, but do not prepare them now. We will do all the prepare when
+ // the configuration is complete.
+ ConfigOptions copy = config_options;
+ copy.invoke_prepare_options = false;
+#ifndef ROCKSDB_LITE
+ if (!config_options.ignore_unknown_options) {
+ // If we are not ignoring unused, get the defaults in case we need to
+ // reset
+ copy.depth = ConfigOptions::kDepthDetailed;
+ copy.delimiter = "; ";
+ GetOptionString(copy, &curr_opts).PermitUncheckedError();
+ }
+#endif // ROCKSDB_LITE
+
+ s = ConfigurableHelper::ConfigureOptions(copy, *this, opts_map, unused);
+ }
+ if (config_options.invoke_prepare_options && s.ok()) {
+ s = PrepareOptions(config_options);
+ }
+#ifndef ROCKSDB_LITE
+ if (!s.ok() && !curr_opts.empty()) {
+ ConfigOptions reset = config_options;
+ reset.ignore_unknown_options = true;
+ reset.invoke_prepare_options = true;
+ reset.ignore_unsupported_options = true;
+ // There are some options to reset from this current error
+ ConfigureFromString(reset, curr_opts).PermitUncheckedError();
+ }
+#endif // ROCKSDB_LITE
+ return s;
+}
+
+Status Configurable::ParseStringOptions(const ConfigOptions& /*config_options*/,
+ const std::string& /*opts_str*/) {
+ return Status::OK();
+}
+
+Status Configurable::ConfigureFromString(const ConfigOptions& config_options,
+ const std::string& opts_str) {
+ Status s;
+ if (!opts_str.empty()) {
+#ifndef ROCKSDB_LITE
+ if (opts_str.find(';') != std::string::npos ||
+ opts_str.find('=') != std::string::npos) {
+ std::unordered_map<std::string, std::string> opt_map;
+ s = StringToMap(opts_str, &opt_map);
+ if (s.ok()) {
+ s = ConfigureFromMap(config_options, opt_map, nullptr);
+ }
+ } else {
+#endif // ROCKSDB_LITE
+ s = ParseStringOptions(config_options, opts_str);
+ if (s.ok() && config_options.invoke_prepare_options) {
+ s = PrepareOptions(config_options);
+ }
+#ifndef ROCKSDB_LITE
+ }
+#endif // ROCKSDB_LITE
+ } else if (config_options.invoke_prepare_options) {
+ s = PrepareOptions(config_options);
+ } else {
+ s = Status::OK();
+ }
+ return s;
+}
+
+#ifndef ROCKSDB_LITE
+/**
+ * Sets the value of the named property to the input value, returning OK on
+ * succcess.
+ */
+Status Configurable::ConfigureOption(const ConfigOptions& config_options,
+ const std::string& name,
+ const std::string& value) {
+ return ConfigurableHelper::ConfigureSingleOption(config_options, *this, name,
+ value);
+}
+
+/**
+ * Looks for the named option amongst the options for this type and sets
+ * the value for it to be the input value.
+ * If the name was found, found_option will be set to true and the resulting
+ * status should be returned.
+ */
+
+Status Configurable::ParseOption(const ConfigOptions& config_options,
+ const OptionTypeInfo& opt_info,
+ const std::string& opt_name,
+ const std::string& opt_value, void* opt_ptr) {
+ if (opt_info.IsMutable()) {
+ if (config_options.mutable_options_only) {
+ // This option is mutable. Treat all of its children as mutable as well
+ ConfigOptions copy = config_options;
+ copy.mutable_options_only = false;
+ return opt_info.Parse(copy, opt_name, opt_value, opt_ptr);
+ } else {
+ return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr);
+ }
+ } else if (config_options.mutable_options_only) {
+ return Status::InvalidArgument("Option not changeable: " + opt_name);
+ } else {
+ return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr);
+ }
+}
+
+#endif // ROCKSDB_LITE
+
+Status ConfigurableHelper::ConfigureOptions(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ std::unordered_map<std::string, std::string>* unused) {
+ std::unordered_map<std::string, std::string> remaining = opts_map;
+ Status s = Status::OK();
+ if (!opts_map.empty()) {
+#ifndef ROCKSDB_LITE
+ for (const auto& iter : configurable.options_) {
+ if (iter.type_map != nullptr) {
+ s = ConfigureSomeOptions(config_options, configurable, *(iter.type_map),
+ &remaining, iter.opt_ptr);
+ if (remaining.empty()) { // Are there more options left?
+ break;
+ } else if (!s.ok()) {
+ break;
+ }
+ }
+ }
+#else
+ (void)configurable;
+ if (!config_options.ignore_unknown_options) {
+ s = Status::NotSupported("ConfigureFromMap not supported in LITE mode");
+ }
+#endif // ROCKSDB_LITE
+ }
+ if (unused != nullptr && !remaining.empty()) {
+ unused->insert(remaining.begin(), remaining.end());
+ }
+ if (config_options.ignore_unknown_options) {
+ s = Status::OK();
+ } else if (s.ok() && unused == nullptr && !remaining.empty()) {
+ s = Status::NotFound("Could not find option: ", remaining.begin()->first);
+ }
+ return s;
+}
+
+#ifndef ROCKSDB_LITE
+/**
+ * Updates the object with the named-value property values, returning OK on
+ * succcess. Any properties that were found are removed from the options list;
+ * upon return only options that were not found in this opt_map remain.
+
+ * Returns:
+ * - OK if ignore_unknown_options is set
+ * - InvalidArgument, if any option was invalid
+ * - NotSupported, if any option is unsupported and ignore_unsupported_options
+ is OFF
+ * - OK, if no option was invalid or not supported (or ignored)
+ */
+Status ConfigurableHelper::ConfigureSomeOptions(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_map,
+ std::unordered_map<std::string, std::string>* options, void* opt_ptr) {
+ Status result = Status::OK(); // The last non-OK result (if any)
+ Status notsup = Status::OK(); // The last NotSupported result (if any)
+ std::string elem_name;
+ int found = 1;
+ std::unordered_set<std::string> unsupported;
+ // While there are unused properties and we processed at least one,
+ // go through the remaining unused properties and attempt to configure them.
+ while (found > 0 && !options->empty()) {
+ found = 0;
+ notsup = Status::OK();
+ for (auto it = options->begin(); it != options->end();) {
+ const std::string& opt_name = configurable.GetOptionName(it->first);
+ const std::string& opt_value = it->second;
+ const auto opt_info =
+ OptionTypeInfo::Find(opt_name, type_map, &elem_name);
+ if (opt_info == nullptr) { // Did not find the option. Skip it
+ ++it;
+ } else {
+ Status s = ConfigureOption(config_options, configurable, *opt_info,
+ opt_name, elem_name, opt_value, opt_ptr);
+ if (s.IsNotFound()) {
+ ++it;
+ } else if (s.IsNotSupported()) {
+ notsup = s;
+ unsupported.insert(it->first);
+ ++it; // Skip it for now
+ } else {
+ found++;
+ it = options->erase(it);
+ if (!s.ok()) {
+ result = s;
+ }
+ }
+ }
+ } // End for all remaining options
+ } // End while found one or options remain
+
+ // Now that we have been through the list, remove any unsupported
+ for (auto u : unsupported) {
+ auto it = options->find(u);
+ if (it != options->end()) {
+ options->erase(it);
+ }
+ }
+ if (config_options.ignore_unknown_options) {
+ if (!result.ok()) result.PermitUncheckedError();
+ if (!notsup.ok()) notsup.PermitUncheckedError();
+ return Status::OK();
+ } else if (!result.ok()) {
+ if (!notsup.ok()) notsup.PermitUncheckedError();
+ return result;
+ } else if (config_options.ignore_unsupported_options) {
+ if (!notsup.ok()) notsup.PermitUncheckedError();
+ return Status::OK();
+ } else {
+ return notsup;
+ }
+}
+
+Status ConfigurableHelper::ConfigureSingleOption(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const std::string& name, const std::string& value) {
+ const std::string& opt_name = configurable.GetOptionName(name);
+ std::string elem_name;
+ void* opt_ptr = nullptr;
+ const auto opt_info =
+ FindOption(configurable.options_, opt_name, &elem_name, &opt_ptr);
+ if (opt_info == nullptr) {
+ return Status::NotFound("Could not find option: ", name);
+ } else {
+ return ConfigureOption(config_options, configurable, *opt_info, opt_name,
+ elem_name, value, opt_ptr);
+ }
+}
+Status ConfigurableHelper::ConfigureCustomizableOption(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const OptionTypeInfo& opt_info, const std::string& opt_name,
+ const std::string& name, const std::string& value, void* opt_ptr) {
+ Customizable* custom = opt_info.AsRawPointer<Customizable>(opt_ptr);
+ ConfigOptions copy = config_options;
+ if (opt_info.IsMutable()) {
+ // This option is mutable. Pass that property on to any subsequent calls
+ copy.mutable_options_only = false;
+ }
+
+ if (opt_info.IsMutable() || !config_options.mutable_options_only) {
+ // Either the option is mutable, or we are processing all of the options
+ if (opt_name == name || name == OptionTypeInfo::kIdPropName() ||
+ EndsWith(opt_name, OptionTypeInfo::kIdPropSuffix())) {
+ return configurable.ParseOption(copy, opt_info, name, value, opt_ptr);
+ } else if (value.empty()) {
+ return Status::OK();
+ } else if (custom == nullptr || !StartsWith(name, custom->GetId() + ".")) {
+ return configurable.ParseOption(copy, opt_info, name, value, opt_ptr);
+ } else if (value.find("=") != std::string::npos) {
+ return custom->ConfigureFromString(copy, value);
+ } else {
+ return custom->ConfigureOption(copy, name, value);
+ }
+ } else {
+ // We are processing immutable options, which means that we cannot change
+ // the Customizable object itself, but could change its mutable properties.
+ // Check to make sure that nothing is trying to change the Customizable
+ if (custom == nullptr) {
+ // We do not have a Customizable to configure. This is OK if the
+ // value is empty (nothing being configured) but an error otherwise
+ if (value.empty()) {
+ return Status::OK();
+ } else {
+ return Status::InvalidArgument("Option not changeable: " + opt_name);
+ }
+ } else if (EndsWith(opt_name, OptionTypeInfo::kIdPropSuffix()) ||
+ name == OptionTypeInfo::kIdPropName()) {
+ // We have a property of the form "id=value" or "table.id=value"
+ // This is OK if we ID/value matches the current customizable object
+ if (custom->GetId() == value) {
+ return Status::OK();
+ } else {
+ return Status::InvalidArgument("Option not changeable: " + opt_name);
+ }
+ } else if (opt_name == name) {
+ // The properties are of one of forms:
+ // name = { id = id; prop1 = value1; ... }
+ // name = { prop1=value1; prop2=value2; ... }
+ // name = ID
+ // Convert the value to a map and extract the ID
+ // If the ID does not match that of the current customizable, return an
+ // error. Otherwise, update the current customizable via the properties
+ // map
+ std::unordered_map<std::string, std::string> props;
+ std::string id;
+ Status s =
+ Configurable::GetOptionsMap(value, custom->GetId(), &id, &props);
+ if (!s.ok()) {
+ return s;
+ } else if (custom->GetId() != id) {
+ return Status::InvalidArgument("Option not changeable: " + opt_name);
+ } else if (props.empty()) {
+ return Status::OK();
+ } else {
+ return custom->ConfigureFromMap(copy, props);
+ }
+ } else {
+ // Attempting to configure one of the properties of the customizable
+ // Let it through
+ return custom->ConfigureOption(copy, name, value);
+ }
+ }
+}
+
+Status ConfigurableHelper::ConfigureOption(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const OptionTypeInfo& opt_info, const std::string& opt_name,
+ const std::string& name, const std::string& value, void* opt_ptr) {
+ if (opt_info.IsCustomizable()) {
+ return ConfigureCustomizableOption(config_options, configurable, opt_info,
+ opt_name, name, value, opt_ptr);
+ } else if (opt_name == name) {
+ return configurable.ParseOption(config_options, opt_info, opt_name, value,
+ opt_ptr);
+ } else if (opt_info.IsStruct() || opt_info.IsConfigurable()) {
+ return configurable.ParseOption(config_options, opt_info, name, value,
+ opt_ptr);
+ } else {
+ return Status::NotFound("Could not find option: ", name);
+ }
+}
+#endif // ROCKSDB_LITE
+
+//*******************************************************************************
+//
+// Methods for Converting Options into strings
+//
+//*******************************************************************************
+
+Status Configurable::GetOptionString(const ConfigOptions& config_options,
+ std::string* result) const {
+ assert(result);
+ result->clear();
+#ifndef ROCKSDB_LITE
+ return ConfigurableHelper::SerializeOptions(config_options, *this, "",
+ result);
+#else
+ (void)config_options;
+ return Status::NotSupported("GetOptionString not supported in LITE mode");
+#endif // ROCKSDB_LITE
+}
+
+#ifndef ROCKSDB_LITE
+std::string Configurable::ToString(const ConfigOptions& config_options,
+ const std::string& prefix) const {
+ std::string result = SerializeOptions(config_options, prefix);
+ if (result.empty() || result.find('=') == std::string::npos) {
+ return result;
+ } else {
+ return "{" + result + "}";
+ }
+}
+
+std::string Configurable::SerializeOptions(const ConfigOptions& config_options,
+ const std::string& header) const {
+ std::string result;
+ Status s = ConfigurableHelper::SerializeOptions(config_options, *this, header,
+ &result);
+ assert(s.ok());
+ return result;
+}
+
+Status Configurable::GetOption(const ConfigOptions& config_options,
+ const std::string& name,
+ std::string* value) const {
+ return ConfigurableHelper::GetOption(config_options, *this,
+ GetOptionName(name), value);
+}
+
+Status ConfigurableHelper::GetOption(const ConfigOptions& config_options,
+ const Configurable& configurable,
+ const std::string& short_name,
+ std::string* value) {
+ // Look for option directly
+ assert(value);
+ value->clear();
+
+ std::string opt_name;
+ void* opt_ptr = nullptr;
+ const auto opt_info =
+ FindOption(configurable.options_, short_name, &opt_name, &opt_ptr);
+ if (opt_info != nullptr) {
+ ConfigOptions embedded = config_options;
+ embedded.delimiter = ";";
+ if (short_name == opt_name) {
+ return opt_info->Serialize(embedded, opt_name, opt_ptr, value);
+ } else if (opt_info->IsStruct()) {
+ return opt_info->Serialize(embedded, opt_name, opt_ptr, value);
+ } else if (opt_info->IsConfigurable()) {
+ auto const* config = opt_info->AsRawPointer<Configurable>(opt_ptr);
+ if (config != nullptr) {
+ return config->GetOption(embedded, opt_name, value);
+ }
+ }
+ }
+ return Status::NotFound("Cannot find option: ", short_name);
+}
+
+Status ConfigurableHelper::SerializeOptions(const ConfigOptions& config_options,
+ const Configurable& configurable,
+ const std::string& prefix,
+ std::string* result) {
+ assert(result);
+ for (auto const& opt_iter : configurable.options_) {
+ if (opt_iter.type_map != nullptr) {
+ for (const auto& map_iter : *(opt_iter.type_map)) {
+ const auto& opt_name = map_iter.first;
+ const auto& opt_info = map_iter.second;
+ if (opt_info.ShouldSerialize()) {
+ std::string value;
+ Status s;
+ if (!config_options.mutable_options_only) {
+ s = opt_info.Serialize(config_options, prefix + opt_name,
+ opt_iter.opt_ptr, &value);
+ } else if (opt_info.IsMutable()) {
+ ConfigOptions copy = config_options;
+ copy.mutable_options_only = false;
+ s = opt_info.Serialize(copy, prefix + opt_name, opt_iter.opt_ptr,
+ &value);
+ } else if (opt_info.IsConfigurable()) {
+ // If it is a Configurable and we are either printing all of the
+ // details or not printing only the name, this option should be
+ // included in the list
+ if (config_options.IsDetailed() ||
+ !opt_info.IsEnabled(OptionTypeFlags::kStringNameOnly)) {
+ s = opt_info.Serialize(config_options, prefix + opt_name,
+ opt_iter.opt_ptr, &value);
+ }
+ }
+ if (!s.ok()) {
+ return s;
+ } else if (!value.empty()) {
+ // <prefix><opt_name>=<value><delimiter>
+ result->append(prefix + opt_name + "=" + value +
+ config_options.delimiter);
+ }
+ }
+ }
+ }
+ }
+ return Status::OK();
+}
+#endif // ROCKSDB_LITE
+
+//********************************************************************************
+//
+// Methods for listing the options from Configurables
+//
+//********************************************************************************
+#ifndef ROCKSDB_LITE
+Status Configurable::GetOptionNames(
+ const ConfigOptions& config_options,
+ std::unordered_set<std::string>* result) const {
+ assert(result);
+ return ConfigurableHelper::ListOptions(config_options, *this, "", result);
+}
+
+Status ConfigurableHelper::ListOptions(
+ const ConfigOptions& config_options, const Configurable& configurable,
+ const std::string& prefix, std::unordered_set<std::string>* result) {
+ Status status;
+ for (auto const& opt_iter : configurable.options_) {
+ if (opt_iter.type_map != nullptr) {
+ for (const auto& map_iter : *(opt_iter.type_map)) {
+ const auto& opt_name = map_iter.first;
+ const auto& opt_info = map_iter.second;
+ // If the option is no longer used in rocksdb and marked as deprecated,
+ // we skip it in the serialization.
+ if (!opt_info.IsDeprecated() && !opt_info.IsAlias()) {
+ if (!config_options.mutable_options_only) {
+ result->emplace(prefix + opt_name);
+ } else if (opt_info.IsMutable()) {
+ result->emplace(prefix + opt_name);
+ }
+ }
+ }
+ }
+ }
+ return status;
+}
+#endif // ROCKSDB_LITE
+
+//*******************************************************************************
+//
+// Methods for Comparing Configurables
+//
+//*******************************************************************************
+
+bool Configurable::AreEquivalent(const ConfigOptions& config_options,
+ const Configurable* other,
+ std::string* name) const {
+ assert(name);
+ name->clear();
+ if (this == other || config_options.IsCheckDisabled()) {
+ return true;
+ } else if (other != nullptr) {
+#ifndef ROCKSDB_LITE
+ return ConfigurableHelper::AreEquivalent(config_options, *this, *other,
+ name);
+#else
+ return true;
+#endif // ROCKSDB_LITE
+ } else {
+ return false;
+ }
+}
+
+#ifndef ROCKSDB_LITE
+bool Configurable::OptionsAreEqual(const ConfigOptions& config_options,
+ const OptionTypeInfo& opt_info,
+ const std::string& opt_name,
+ const void* const this_ptr,
+ const void* const that_ptr,
+ std::string* mismatch) const {
+ if (opt_info.AreEqual(config_options, opt_name, this_ptr, that_ptr,
+ mismatch)) {
+ return true;
+ } else if (opt_info.AreEqualByName(config_options, opt_name, this_ptr,
+ that_ptr)) {
+ *mismatch = "";
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool ConfigurableHelper::AreEquivalent(const ConfigOptions& config_options,
+ const Configurable& this_one,
+ const Configurable& that_one,
+ std::string* mismatch) {
+ assert(mismatch != nullptr);
+ for (auto const& o : this_one.options_) {
+ const auto this_offset = this_one.GetOptionsPtr(o.name);
+ const auto that_offset = that_one.GetOptionsPtr(o.name);
+ if (this_offset != that_offset) {
+ if (this_offset == nullptr || that_offset == nullptr) {
+ return false;
+ } else if (o.type_map != nullptr) {
+ for (const auto& map_iter : *(o.type_map)) {
+ const auto& opt_info = map_iter.second;
+ if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) {
+ if (!config_options.mutable_options_only) {
+ if (!this_one.OptionsAreEqual(config_options, opt_info,
+ map_iter.first, this_offset,
+ that_offset, mismatch)) {
+ return false;
+ }
+ } else if (opt_info.IsMutable()) {
+ ConfigOptions copy = config_options;
+ copy.mutable_options_only = false;
+ if (!this_one.OptionsAreEqual(copy, opt_info, map_iter.first,
+ this_offset, that_offset,
+ mismatch)) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return true;
+}
+#endif // ROCKSDB_LITE
+
+Status Configurable::GetOptionsMap(
+ const std::string& value, const std::string& default_id, std::string* id,
+ std::unordered_map<std::string, std::string>* props) {
+ assert(id);
+ assert(props);
+ Status status;
+ if (value.empty() || value == kNullptrString) {
+ *id = default_id;
+ } else if (value.find('=') == std::string::npos) {
+ *id = value;
+#ifndef ROCKSDB_LITE
+ } else {
+ status = StringToMap(value, props);
+ if (!status.ok()) { // There was an error creating the map.
+ *id = value; // Treat the value as id
+ props->clear(); // Clear the properties
+ status = Status::OK(); // and ignore the error
+ } else {
+ auto iter = props->find(OptionTypeInfo::kIdPropName());
+ if (iter != props->end()) {
+ *id = iter->second;
+ props->erase(iter);
+ if (*id == kNullptrString) {
+ id->clear();
+ }
+ } else if (!default_id.empty()) {
+ *id = default_id;
+ } else { // No id property and no default
+ *id = value; // Treat the value as id
+ props->clear(); // Clear the properties
+ }
+ }
+#else
+ } else {
+ *id = value;
+ props->clear();
+#endif
+ }
+ return status;
+}
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/configurable_helper.h b/src/rocksdb/options/configurable_helper.h
new file mode 100644
index 000000000..0f5f918cb
--- /dev/null
+++ b/src/rocksdb/options/configurable_helper.h
@@ -0,0 +1,187 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <map>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "rocksdb/configurable.h"
+#include "rocksdb/convenience.h"
+
+namespace ROCKSDB_NAMESPACE {
+// Helper class defining static methods for supporting the Configurable
+// class. The purpose of this class is to keep the Configurable class
+// as tight as possible and provide methods for doing the actual work
+// of configuring the objects.
+class ConfigurableHelper {
+ public:
+ // Configures the input Configurable object based on the parameters.
+ // On successful completion, the Configurable is updated with the settings
+ // from the opt_map.
+ //
+ // The acceptable values of the name/value pairs are documented with the
+ // specific class/instance.
+ //
+ // @param config_options Controls how the arguments are processed.
+ // @param opt_map Name/value pairs of the options to update
+ // @param unused If specified, this value will return the name/value
+ // pairs from opt_map that were NotFound for this object.
+ // @return OK If all values in the map were successfully updated
+ // @return NotFound If any of the names in the opt_map were not valid
+ // for this object. If unused is specified, it will contain the
+ // collection of NotFound entries
+ // @return NotSupported If any of the names are valid but the object does
+ // not know how to convert the value. This can happen if, for example,
+ // there is some nested Configurable that cannot be created.
+ // @return InvalidArgument If any of the values cannot be successfully
+ // parsed. This can also be returned if PrepareOptions encounters an
+ // error.
+ static Status ConfigureOptions(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const std::unordered_map<std::string, std::string>& options,
+ std::unordered_map<std::string, std::string>* unused);
+
+#ifndef ROCKSDB_LITE
+ // Internal method to configure a set of options for this object.
+ // Classes may override this value to change its behavior.
+ // @param config_options Controls how the options are being configured
+ // @param type_name The name that was registered for this set of options
+ // @param type_map The map of options for this name
+ // @param opt_ptr Pointer to the object being configured for this option set.
+ // @param options The option name/values being updated. On return, any
+ // option that was found is removed from the list.
+ // @return OK If all of the options were successfully updated.
+ // @return InvalidArgument If an option was found but the value could not
+ // be updated.
+ // @return NotFound If an option name was not found in type_mape
+ // @return NotSupported If the option was found but no rule for converting
+ // the value could be found.
+ static Status ConfigureSomeOptions(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_map,
+ std::unordered_map<std::string, std::string>* options, void* opt_ptr);
+
+ // Configures a single option in the input Configurable.
+ // This method will look through the set of option names for this
+ // Configurable searching for one with the input name. If such an option
+ // is found, it will be configured via the input value.
+ //
+ // @param config_options Controls how the option is being configured
+ // @param configurable The object to configure
+ // @param name For options with sub-options (like Structs or
+ // Configurables),
+ // this value may be the name of the sub-field of the option being
+ // updated. For example, if the option is
+ // "compaction_options_fifo.allow_compaction", then field name would be
+ // "allow_compaction". For most options, field_name and opt_name will be
+ // equivalent.
+ // @param value The new value for this option.
+ // @param See ConfigureOptions for the possible return values
+ static Status ConfigureSingleOption(const ConfigOptions& config_options,
+ Configurable& configurable,
+ const std::string& name,
+ const std::string& value);
+
+ // Configures the option referenced by opt_info for this configurable
+ // This method configures the option based on opt_info for the input
+ // configurable.
+ // @param config_options Controls how the option is being configured
+ // @param configurable The object to configure
+ // @param opt_name The full option name
+ // @param name For options with sub-options (like Structs or
+ // Configurables),
+ // this value may be the name of the sub-field of the option being
+ // updated. For example, if the option is
+ // "compaction_options_fifo.allow_compaction", then field name would be
+ // "allow_compaction". For most options, field_name and opt_name will be
+ // equivalent.
+ // @param value The new value for this option.
+ // @param See ConfigureOptions for the possible return values
+ static Status ConfigureOption(const ConfigOptions& config_options,
+ Configurable& configurable,
+ const OptionTypeInfo& opt_info,
+ const std::string& opt_name,
+ const std::string& name,
+ const std::string& value, void* opt_ptr);
+
+ // Returns the value of the option associated with the input name
+ // This method is the functional inverse of ConfigureOption
+ // @param config_options Controls how the value is returned
+ // @param configurable The object from which to get the option.
+ // @param name The name of the option to return a value for.
+ // @param value The returned value associated with the named option.
+ // Note that value will be only the serialized version
+ // of the option and not "name=value"
+ // @return OK If the named field was successfully updated to value.
+ // @return NotFound If the name is not valid for this object.
+ // @param InvalidArgument If the name is valid for this object but
+ // its value cannot be serialized.
+ static Status GetOption(const ConfigOptions& config_options,
+ const Configurable& configurable,
+ const std::string& name, std::string* value);
+
+ // Serializes the input Configurable into the output result.
+ // This is the inverse of ConfigureOptions
+ // @param config_options Controls how serialization happens.
+ // @param configurable The object to serialize
+ // @param prefix A prefix to add to the each option as it is serialized.
+ // @param result The string representation of the configurable.
+ // @return OK If the options for this object wer successfully serialized.
+ // @return InvalidArgument If one or more of the options could not be
+ // serialized.
+ static Status SerializeOptions(const ConfigOptions& config_options,
+ const Configurable& configurable,
+ const std::string& prefix,
+ std::string* result);
+
+ // Internal method to list the option names for this object.
+ // Classes may override this value to change its behavior.
+ // @see ListOptions for more details
+ static Status ListOptions(const ConfigOptions& config_options,
+ const Configurable& configurable,
+ const std::string& prefix,
+ std::unordered_set<std::string>* result);
+
+ // Checks to see if the two configurables are equivalent to one other.
+ // This method assumes that the two objects are of the same class.
+ // @param config_options Controls how the options are compared.
+ // @param this_one The object to compare to.
+ // @param that_one The other object being compared.
+ // @param mismatch If the objects do not match, this parameter contains
+ // the name of the option that triggered the match failure.
+ // @param True if the objects match, false otherwise.
+ static bool AreEquivalent(const ConfigOptions& config_options,
+ const Configurable& this_one,
+ const Configurable& that_one,
+ std::string* mismatch);
+
+ private:
+ // Looks for the option specified by name in the RegisteredOptions.
+ // This method traverses the types in the input options vector. If an entry
+ // matching name is found, that entry, opt_name, and pointer are returned.
+ // @param options The vector of options to search through
+ // @param name The name of the option to search for in the OptionType map
+ // @param opt_name If the name was found, this value is set to the option name
+ // associated with the input name/type.
+ // @param opt_ptr If the name was found, this value is set to the option
+ // pointer
+ // in the RegisteredOptions vector associated with this entry
+ // @return A pointer to the OptionTypeInfo from the options if found,
+ // nullptr if the name was not found in the input options
+ static const OptionTypeInfo* FindOption(
+ const std::vector<Configurable::RegisteredOptions>& options,
+ const std::string& name, std::string* opt_name, void** opt_ptr);
+
+ static Status ConfigureCustomizableOption(
+ const ConfigOptions& config_options, Configurable& configurable,
+ const OptionTypeInfo& opt_info, const std::string& opt_name,
+ const std::string& name, const std::string& value, void* opt_ptr);
+#endif // ROCKSDB_LITE
+};
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/configurable_test.cc b/src/rocksdb/options/configurable_test.cc
new file mode 100644
index 000000000..6ec02cf3a
--- /dev/null
+++ b/src/rocksdb/options/configurable_test.cc
@@ -0,0 +1,881 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "options/configurable_test.h"
+
+#include <cctype>
+#include <cinttypes>
+#include <cstring>
+#include <unordered_map>
+
+#include "options/configurable_helper.h"
+#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "rocksdb/configurable.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+
+#ifndef GFLAGS
+bool FLAGS_enable_print = false;
+#else
+#include "util/gflags_compat.h"
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+DEFINE_bool(enable_print, false, "Print options generated to console.");
+#endif // GFLAGS
+
+namespace ROCKSDB_NAMESPACE {
+namespace test {
+class StringLogger : public Logger {
+ public:
+ using Logger::Logv;
+ void Logv(const char* format, va_list ap) override {
+ char buffer[1000];
+ vsnprintf(buffer, sizeof(buffer), format, ap);
+ string_.append(buffer);
+ }
+ const std::string& str() const { return string_; }
+ void clear() { string_.clear(); }
+
+ private:
+ std::string string_;
+};
+static std::unordered_map<std::string, OptionTypeInfo> struct_option_info = {
+#ifndef ROCKSDB_LITE
+ {"struct", OptionTypeInfo::Struct("struct", &simple_option_info, 0,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable)},
+#endif // ROCKSDB_LITE
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> imm_struct_option_info =
+ {
+#ifndef ROCKSDB_LITE
+ {"struct", OptionTypeInfo::Struct("struct", &simple_option_info, 0,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone)},
+#endif // ROCKSDB_LITE
+};
+
+class SimpleConfigurable : public TestConfigurable<Configurable> {
+ public:
+ static SimpleConfigurable* Create(
+ const std::string& name = "simple",
+ int mode = TestConfigMode::kDefaultMode,
+ const std::unordered_map<std::string, OptionTypeInfo>* map =
+ &simple_option_info) {
+ return new SimpleConfigurable(name, mode, map);
+ }
+
+ SimpleConfigurable(const std::string& name, int mode,
+ const std::unordered_map<std::string, OptionTypeInfo>*
+ map = &simple_option_info)
+ : TestConfigurable(name, mode, map) {
+ if ((mode & TestConfigMode::kUniqueMode) != 0) {
+ unique_.reset(SimpleConfigurable::Create("Unique" + name_));
+ RegisterOptions(name_ + "Unique", &unique_, &unique_option_info);
+ }
+ if ((mode & TestConfigMode::kSharedMode) != 0) {
+ shared_.reset(SimpleConfigurable::Create("Shared" + name_));
+ RegisterOptions(name_ + "Shared", &shared_, &shared_option_info);
+ }
+ if ((mode & TestConfigMode::kRawPtrMode) != 0) {
+ pointer_ = SimpleConfigurable::Create("Pointer" + name_);
+ RegisterOptions(name_ + "Pointer", &pointer_, &pointer_option_info);
+ }
+ }
+
+}; // End class SimpleConfigurable
+
+using ConfigTestFactoryFunc = std::function<Configurable*()>;
+
+class ConfigurableTest : public testing::Test {
+ public:
+ ConfigurableTest() { config_options_.invoke_prepare_options = false; }
+
+ ConfigOptions config_options_;
+};
+
+TEST_F(ConfigurableTest, GetOptionsPtrTest) {
+ std::string opt_str;
+ std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
+ ASSERT_NE(configurable->GetOptions<TestOptions>("simple"), nullptr);
+ ASSERT_EQ(configurable->GetOptions<TestOptions>("bad-opt"), nullptr);
+}
+
+TEST_F(ConfigurableTest, ConfigureFromMapTest) {
+ std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
+ auto* opts = configurable->GetOptions<TestOptions>("simple");
+ ASSERT_OK(configurable->ConfigureFromMap(config_options_, {}));
+ ASSERT_NE(opts, nullptr);
+#ifndef ROCKSDB_LITE
+ std::unordered_map<std::string, std::string> options_map = {
+ {"int", "1"}, {"bool", "true"}, {"string", "string"}};
+ ASSERT_OK(configurable->ConfigureFromMap(config_options_, options_map));
+ ASSERT_EQ(opts->i, 1);
+ ASSERT_EQ(opts->b, true);
+ ASSERT_EQ(opts->s, "string");
+#endif
+}
+
+TEST_F(ConfigurableTest, ConfigureFromStringTest) {
+ std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
+ auto* opts = configurable->GetOptions<TestOptions>("simple");
+ ASSERT_OK(configurable->ConfigureFromString(config_options_, ""));
+ ASSERT_NE(opts, nullptr);
+#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
+ ASSERT_OK(configurable->ConfigureFromString(config_options_,
+ "int=1;bool=true;string=s"));
+ ASSERT_EQ(opts->i, 1);
+ ASSERT_EQ(opts->b, true);
+ ASSERT_EQ(opts->s, "s");
+#endif
+}
+
+#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
+TEST_F(ConfigurableTest, ConfigureIgnoreTest) {
+ std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
+ std::unordered_map<std::string, std::string> options_map = {{"unused", "u"}};
+ ConfigOptions ignore = config_options_;
+ ignore.ignore_unknown_options = true;
+ ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map));
+ ASSERT_OK(configurable->ConfigureFromMap(ignore, options_map));
+ ASSERT_NOK(configurable->ConfigureFromString(config_options_, "unused=u"));
+ ASSERT_OK(configurable->ConfigureFromString(ignore, "unused=u"));
+}
+
+TEST_F(ConfigurableTest, ConfigureNestedOptionsTest) {
+ std::unique_ptr<Configurable> base, copy;
+ std::string opt_str;
+ std::string mismatch;
+
+ base.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode));
+ copy.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode));
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "shared={int=10; string=10};"
+ "unique={int=20; string=20};"
+ "pointer={int=30; string=30};"));
+ ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
+ ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+}
+
+TEST_F(ConfigurableTest, GetOptionsTest) {
+ std::unique_ptr<Configurable> simple;
+
+ simple.reset(
+ SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode));
+ int i = 11;
+ for (auto opt : {"", "shared.", "unique.", "pointer."}) {
+ std::string value;
+ std::string expected = std::to_string(i);
+ std::string opt_name = opt;
+ ASSERT_OK(
+ simple->ConfigureOption(config_options_, opt_name + "int", expected));
+ ASSERT_OK(simple->GetOption(config_options_, opt_name + "int", &value));
+ ASSERT_EQ(expected, value);
+ ASSERT_OK(simple->ConfigureOption(config_options_, opt_name + "string",
+ expected));
+ ASSERT_OK(simple->GetOption(config_options_, opt_name + "string", &value));
+ ASSERT_EQ(expected, value);
+
+ ASSERT_NOK(
+ simple->ConfigureOption(config_options_, opt_name + "bad", expected));
+ ASSERT_NOK(simple->GetOption(config_options_, "bad option", &value));
+ ASSERT_TRUE(value.empty());
+ i += 11;
+ }
+}
+
+TEST_F(ConfigurableTest, ConfigureBadOptionsTest) {
+ std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
+ auto* opts = configurable->GetOptions<TestOptions>("simple");
+ ASSERT_NE(opts, nullptr);
+ ASSERT_OK(configurable->ConfigureOption(config_options_, "int", "42"));
+ ASSERT_EQ(opts->i, 42);
+ ASSERT_NOK(configurable->ConfigureOption(config_options_, "int", "fred"));
+ ASSERT_NOK(configurable->ConfigureOption(config_options_, "bool", "fred"));
+ ASSERT_NOK(
+ configurable->ConfigureFromString(config_options_, "int=33;unused=u"));
+ ASSERT_EQ(opts->i, 42);
+}
+
+TEST_F(ConfigurableTest, InvalidOptionTest) {
+ std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
+ std::unordered_map<std::string, std::string> options_map = {
+ {"bad-option", "bad"}};
+ ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map));
+ ASSERT_NOK(
+ configurable->ConfigureFromString(config_options_, "bad-option=bad"));
+ ASSERT_NOK(
+ configurable->ConfigureOption(config_options_, "bad-option", "bad"));
+}
+
+static std::unordered_map<std::string, OptionTypeInfo> validated_option_info = {
+#ifndef ROCKSDB_LITE
+ {"validated",
+ {0, OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+#endif // ROCKSDB_LITE
+};
+static std::unordered_map<std::string, OptionTypeInfo> prepared_option_info = {
+#ifndef ROCKSDB_LITE
+ {"prepared",
+ {0, OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+#endif // ROCKSDB_LITE
+};
+static std::unordered_map<std::string, OptionTypeInfo>
+ dont_prepare_option_info = {
+#ifndef ROCKSDB_LITE
+ {"unique",
+ {0, OptionType::kConfigurable, OptionVerificationType::kNormal,
+ (OptionTypeFlags::kUnique | OptionTypeFlags::kDontPrepare)}},
+
+#endif // ROCKSDB_LITE
+};
+
+class ValidatedConfigurable : public SimpleConfigurable {
+ public:
+ ValidatedConfigurable(const std::string& name, unsigned char mode,
+ bool dont_prepare = false)
+ : SimpleConfigurable(name, TestConfigMode::kDefaultMode),
+ validated(false),
+ prepared(0) {
+ RegisterOptions("Validated", &validated, &validated_option_info);
+ RegisterOptions("Prepared", &prepared, &prepared_option_info);
+ if ((mode & TestConfigMode::kUniqueMode) != 0) {
+ unique_.reset(new ValidatedConfigurable(
+ "Unique" + name_, TestConfigMode::kDefaultMode, false));
+ if (dont_prepare) {
+ RegisterOptions(name_ + "Unique", &unique_, &dont_prepare_option_info);
+ } else {
+ RegisterOptions(name_ + "Unique", &unique_, &unique_option_info);
+ }
+ }
+ }
+
+ Status PrepareOptions(const ConfigOptions& config_options) override {
+ if (++prepared <= 0) {
+ return Status::InvalidArgument("Cannot prepare option");
+ } else {
+ return SimpleConfigurable::PrepareOptions(config_options);
+ }
+ }
+
+ Status ValidateOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts) const override {
+ if (!validated) {
+ return Status::InvalidArgument("Not Validated");
+ } else {
+ return SimpleConfigurable::ValidateOptions(db_opts, cf_opts);
+ }
+ }
+
+ private:
+ bool validated;
+ int prepared;
+};
+
+TEST_F(ConfigurableTest, ValidateOptionsTest) {
+ std::unique_ptr<Configurable> configurable(
+ new ValidatedConfigurable("validated", TestConfigMode::kDefaultMode));
+ ColumnFamilyOptions cf_opts;
+ DBOptions db_opts;
+ ASSERT_OK(
+ configurable->ConfigureOption(config_options_, "validated", "false"));
+ ASSERT_NOK(configurable->ValidateOptions(db_opts, cf_opts));
+ ASSERT_OK(
+ configurable->ConfigureOption(config_options_, "validated", "true"));
+ ASSERT_OK(configurable->ValidateOptions(db_opts, cf_opts));
+}
+
+TEST_F(ConfigurableTest, PrepareOptionsTest) {
+ std::unique_ptr<Configurable> c(
+ new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, false));
+ auto cp = c->GetOptions<int>("Prepared");
+ auto u = c->GetOptions<std::unique_ptr<Configurable>>("SimpleUnique");
+ auto up = u->get()->GetOptions<int>("Prepared");
+ config_options_.invoke_prepare_options = false;
+
+ ASSERT_NE(cp, nullptr);
+ ASSERT_NE(up, nullptr);
+ ASSERT_EQ(*cp, 0);
+ ASSERT_EQ(*up, 0);
+ ASSERT_OK(c->ConfigureFromMap(config_options_, {}));
+ ASSERT_EQ(*cp, 0);
+ ASSERT_EQ(*up, 0);
+ config_options_.invoke_prepare_options = true;
+ ASSERT_OK(c->ConfigureFromMap(config_options_, {}));
+ ASSERT_EQ(*cp, 1);
+ ASSERT_EQ(*up, 1);
+ ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0"));
+ ASSERT_EQ(*up, 2);
+ ASSERT_EQ(*cp, 1);
+
+ ASSERT_NOK(c->ConfigureFromString(config_options_, "prepared=-2"));
+
+ c.reset(
+ new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, true));
+ cp = c->GetOptions<int>("Prepared");
+ u = c->GetOptions<std::unique_ptr<Configurable>>("SimpleUnique");
+ up = u->get()->GetOptions<int>("Prepared");
+
+ ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0"));
+ ASSERT_EQ(*cp, 1);
+ ASSERT_EQ(*up, 0);
+}
+
+TEST_F(ConfigurableTest, CopyObjectTest) {
+ class CopyConfigurable : public Configurable {
+ public:
+ CopyConfigurable() : prepared_(0), validated_(0) {}
+ Status PrepareOptions(const ConfigOptions& options) override {
+ prepared_++;
+ return Configurable::PrepareOptions(options);
+ }
+ Status ValidateOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts) const override {
+ validated_++;
+ return Configurable::ValidateOptions(db_opts, cf_opts);
+ }
+ int prepared_;
+ mutable int validated_;
+ };
+
+ CopyConfigurable c1;
+ ConfigOptions config_options;
+ Options options;
+
+ ASSERT_OK(c1.PrepareOptions(config_options));
+ ASSERT_OK(c1.ValidateOptions(options, options));
+ ASSERT_EQ(c1.prepared_, 1);
+ ASSERT_EQ(c1.validated_, 1);
+ CopyConfigurable c2 = c1;
+ ASSERT_OK(c1.PrepareOptions(config_options));
+ ASSERT_OK(c1.ValidateOptions(options, options));
+ ASSERT_EQ(c2.prepared_, 1);
+ ASSERT_EQ(c2.validated_, 1);
+ ASSERT_EQ(c1.prepared_, 2);
+ ASSERT_EQ(c1.validated_, 2);
+}
+
+TEST_F(ConfigurableTest, MutableOptionsTest) {
+ static std::unordered_map<std::string, OptionTypeInfo> imm_option_info = {
+#ifndef ROCKSDB_LITE
+ {"imm", OptionTypeInfo::Struct("imm", &simple_option_info, 0,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone)},
+#endif // ROCKSDB_LITE
+ };
+
+ class MutableConfigurable : public SimpleConfigurable {
+ public:
+ MutableConfigurable()
+ : SimpleConfigurable("mutable", TestConfigMode::kDefaultMode |
+ TestConfigMode::kUniqueMode |
+ TestConfigMode::kSharedMode) {
+ RegisterOptions("struct", &options_, &struct_option_info);
+ RegisterOptions("imm", &options_, &imm_option_info);
+ }
+ };
+ MutableConfigurable mc;
+ ConfigOptions options = config_options_;
+
+ ASSERT_OK(mc.ConfigureOption(options, "bool", "true"));
+ ASSERT_OK(mc.ConfigureOption(options, "int", "42"));
+ auto* opts = mc.GetOptions<TestOptions>("mutable");
+ ASSERT_NE(opts, nullptr);
+ ASSERT_EQ(opts->i, 42);
+ ASSERT_EQ(opts->b, true);
+ ASSERT_OK(mc.ConfigureOption(options, "struct", "{bool=false;}"));
+ ASSERT_OK(mc.ConfigureOption(options, "imm", "{int=55;}"));
+
+ options.mutable_options_only = true;
+
+ // Now only mutable options should be settable.
+ ASSERT_NOK(mc.ConfigureOption(options, "bool", "true"));
+ ASSERT_OK(mc.ConfigureOption(options, "int", "24"));
+ ASSERT_EQ(opts->i, 24);
+ ASSERT_EQ(opts->b, false);
+ ASSERT_NOK(mc.ConfigureFromString(options, "bool=false;int=33;"));
+ ASSERT_EQ(opts->i, 24);
+ ASSERT_EQ(opts->b, false);
+
+ // Setting options through an immutable struct fails
+ ASSERT_NOK(mc.ConfigureOption(options, "imm", "{int=55;}"));
+ ASSERT_NOK(mc.ConfigureOption(options, "imm.int", "55"));
+ ASSERT_EQ(opts->i, 24);
+ ASSERT_EQ(opts->b, false);
+
+ // Setting options through an mutable struct succeeds
+ ASSERT_OK(mc.ConfigureOption(options, "struct", "{int=44;}"));
+ ASSERT_EQ(opts->i, 44);
+ ASSERT_OK(mc.ConfigureOption(options, "struct.int", "55"));
+ ASSERT_EQ(opts->i, 55);
+
+ // Setting nested immutable configurable options fail
+ ASSERT_NOK(mc.ConfigureOption(options, "shared", "{bool=true;}"));
+ ASSERT_NOK(mc.ConfigureOption(options, "shared.bool", "true"));
+
+ // Setting nested mutable configurable options succeeds
+ ASSERT_OK(mc.ConfigureOption(options, "unique", "{bool=true}"));
+ ASSERT_OK(mc.ConfigureOption(options, "unique.bool", "true"));
+}
+
+TEST_F(ConfigurableTest, DeprecatedOptionsTest) {
+ static std::unordered_map<std::string, OptionTypeInfo>
+ deprecated_option_info = {
+ {"deprecated",
+ {offsetof(struct TestOptions, b), OptionType::kBoolean,
+ OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}};
+ std::unique_ptr<Configurable> orig;
+ orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode,
+ &deprecated_option_info));
+ auto* opts = orig->GetOptions<TestOptions>("simple");
+ ASSERT_NE(opts, nullptr);
+ opts->d = true;
+ ASSERT_OK(orig->ConfigureOption(config_options_, "deprecated", "false"));
+ ASSERT_TRUE(opts->d);
+ ASSERT_OK(orig->ConfigureFromString(config_options_, "deprecated=false"));
+ ASSERT_TRUE(opts->d);
+}
+
+TEST_F(ConfigurableTest, AliasOptionsTest) {
+ static std::unordered_map<std::string, OptionTypeInfo> alias_option_info = {
+ {"bool",
+ {offsetof(struct TestOptions, b), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"alias",
+ {offsetof(struct TestOptions, b), OptionType::kBoolean,
+ OptionVerificationType::kAlias, OptionTypeFlags::kNone, 0}}};
+ std::unique_ptr<Configurable> orig;
+ orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode,
+ &alias_option_info));
+ auto* opts = orig->GetOptions<TestOptions>("simple");
+ ASSERT_NE(opts, nullptr);
+ ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false"));
+ ASSERT_FALSE(opts->b);
+ ASSERT_OK(orig->ConfigureOption(config_options_, "alias", "true"));
+ ASSERT_TRUE(opts->b);
+ std::string opts_str;
+ ASSERT_OK(orig->GetOptionString(config_options_, &opts_str));
+ ASSERT_EQ(opts_str.find("alias"), std::string::npos);
+
+ ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false"));
+ ASSERT_FALSE(opts->b);
+ ASSERT_OK(orig->GetOption(config_options_, "alias", &opts_str));
+ ASSERT_EQ(opts_str, "false");
+}
+
+TEST_F(ConfigurableTest, NestedUniqueConfigTest) {
+ std::unique_ptr<Configurable> simple;
+ simple.reset(
+ SimpleConfigurable::Create("Outer", TestConfigMode::kAllOptMode));
+ const auto outer = simple->GetOptions<TestOptions>("Outer");
+ const auto unique =
+ simple->GetOptions<std::unique_ptr<Configurable>>("OuterUnique");
+ ASSERT_NE(outer, nullptr);
+ ASSERT_NE(unique, nullptr);
+ ASSERT_OK(
+ simple->ConfigureFromString(config_options_, "int=24;string=outer"));
+ ASSERT_OK(simple->ConfigureFromString(config_options_,
+ "unique={int=42;string=nested}"));
+ const auto inner = unique->get()->GetOptions<TestOptions>("UniqueOuter");
+ ASSERT_NE(inner, nullptr);
+ ASSERT_EQ(outer->i, 24);
+ ASSERT_EQ(outer->s, "outer");
+ ASSERT_EQ(inner->i, 42);
+ ASSERT_EQ(inner->s, "nested");
+}
+
+TEST_F(ConfigurableTest, NestedSharedConfigTest) {
+ std::unique_ptr<Configurable> simple;
+ simple.reset(SimpleConfigurable::Create(
+ "Outer", TestConfigMode::kDefaultMode | TestConfigMode::kSharedMode));
+ ASSERT_OK(
+ simple->ConfigureFromString(config_options_, "int=24;string=outer"));
+ ASSERT_OK(simple->ConfigureFromString(config_options_,
+ "shared={int=42;string=nested}"));
+ const auto outer = simple->GetOptions<TestOptions>("Outer");
+ const auto shared =
+ simple->GetOptions<std::shared_ptr<Configurable>>("OuterShared");
+ ASSERT_NE(outer, nullptr);
+ ASSERT_NE(shared, nullptr);
+ const auto inner = shared->get()->GetOptions<TestOptions>("SharedOuter");
+ ASSERT_NE(inner, nullptr);
+ ASSERT_EQ(outer->i, 24);
+ ASSERT_EQ(outer->s, "outer");
+ ASSERT_EQ(inner->i, 42);
+ ASSERT_EQ(inner->s, "nested");
+}
+
+TEST_F(ConfigurableTest, NestedRawConfigTest) {
+ std::unique_ptr<Configurable> simple;
+ simple.reset(SimpleConfigurable::Create(
+ "Outer", TestConfigMode::kDefaultMode | TestConfigMode::kRawPtrMode));
+ ASSERT_OK(
+ simple->ConfigureFromString(config_options_, "int=24;string=outer"));
+ ASSERT_OK(simple->ConfigureFromString(config_options_,
+ "pointer={int=42;string=nested}"));
+ const auto outer = simple->GetOptions<TestOptions>("Outer");
+ const auto pointer = simple->GetOptions<Configurable*>("OuterPointer");
+ ASSERT_NE(outer, nullptr);
+ ASSERT_NE(pointer, nullptr);
+ const auto inner = (*pointer)->GetOptions<TestOptions>("PointerOuter");
+ ASSERT_NE(inner, nullptr);
+ ASSERT_EQ(outer->i, 24);
+ ASSERT_EQ(outer->s, "outer");
+ ASSERT_EQ(inner->i, 42);
+ ASSERT_EQ(inner->s, "nested");
+}
+
+TEST_F(ConfigurableTest, MatchesTest) {
+ std::string mismatch;
+ std::unique_ptr<Configurable> base, copy;
+ base.reset(SimpleConfigurable::Create(
+ "simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode));
+ copy.reset(SimpleConfigurable::Create(
+ "simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode));
+ ASSERT_OK(base->ConfigureFromString(
+ config_options_,
+ "int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}"));
+ ASSERT_OK(copy->ConfigureFromString(
+ config_options_,
+ "int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}"));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_OK(base->ConfigureOption(config_options_, "shared", "int=44"));
+ ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_EQ(mismatch, "shared.int");
+ std::string c1value, c2value;
+ ASSERT_OK(base->GetOption(config_options_, mismatch, &c1value));
+ ASSERT_OK(copy->GetOption(config_options_, mismatch, &c2value));
+ ASSERT_NE(c1value, c2value);
+}
+
+static Configurable* SimpleStructFactory() {
+ return SimpleConfigurable::Create(
+ "simple-struct", TestConfigMode::kDefaultMode, &struct_option_info);
+}
+
+TEST_F(ConfigurableTest, ConfigureStructTest) {
+ std::unique_ptr<Configurable> base(SimpleStructFactory());
+ std::unique_ptr<Configurable> copy(SimpleStructFactory());
+ std::string opt_str, value;
+ std::string mismatch;
+ std::unordered_set<std::string> names;
+
+ ASSERT_OK(
+ base->ConfigureFromString(config_options_, "struct={int=10; string=10}"));
+ ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
+ ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_OK(base->GetOptionNames(config_options_, &names));
+ ASSERT_EQ(names.size(), 1);
+ ASSERT_EQ(*(names.begin()), "struct");
+ ASSERT_OK(
+ base->ConfigureFromString(config_options_, "struct={int=20; string=20}"));
+ ASSERT_OK(base->GetOption(config_options_, "struct", &value));
+ ASSERT_OK(copy->ConfigureOption(config_options_, "struct", value));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+
+ ASSERT_NOK(base->ConfigureFromString(config_options_,
+ "struct={int=10; string=10; bad=11}"));
+ ASSERT_OK(base->ConfigureOption(config_options_, "struct.int", "42"));
+ ASSERT_NOK(base->ConfigureOption(config_options_, "struct.bad", "42"));
+ ASSERT_NOK(base->GetOption(config_options_, "struct.bad", &value));
+ ASSERT_OK(base->GetOption(config_options_, "struct.int", &value));
+ ASSERT_EQ(value, "42");
+}
+
+TEST_F(ConfigurableTest, ConfigurableEnumTest) {
+ std::unique_ptr<Configurable> base, copy;
+ base.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode));
+ copy.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode));
+
+ std::string opts_str;
+ std::string mismatch;
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "enum=B"));
+ ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_OK(base->GetOptionString(config_options_, &opts_str));
+ ASSERT_OK(copy->ConfigureFromString(config_options_, opts_str));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_NOK(base->ConfigureOption(config_options_, "enum", "bad"));
+ ASSERT_NOK(base->ConfigureOption(config_options_, "unknown", "bad"));
+}
+
+#ifndef ROCKSDB_LITE
+static std::unordered_map<std::string, OptionTypeInfo> noserialize_option_info =
+ {
+ {"int",
+ {offsetof(struct TestOptions, i), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kDontSerialize}},
+};
+
+TEST_F(ConfigurableTest, TestNoSerialize) {
+ std::unique_ptr<Configurable> base;
+ base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode,
+ &noserialize_option_info));
+ std::string opts_str, value;
+ ASSERT_OK(base->ConfigureFromString(config_options_, "int=10"));
+ ASSERT_OK(base->GetOptionString(config_options_, &opts_str));
+ ASSERT_EQ(opts_str, "");
+ ASSERT_NOK(base->GetOption(config_options_, "int", &value));
+}
+
+TEST_F(ConfigurableTest, TestNoCompare) {
+ std::unordered_map<std::string, OptionTypeInfo> nocomp_option_info = {
+ {"int",
+ {offsetof(struct TestOptions, i), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}},
+ };
+ std::unordered_map<std::string, OptionTypeInfo> normal_option_info = {
+ {"int",
+ {offsetof(struct TestOptions, i), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ };
+
+ std::unique_ptr<Configurable> base, copy;
+ base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode,
+ &nocomp_option_info));
+ copy.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode,
+ &normal_option_info));
+ ASSERT_OK(base->ConfigureFromString(config_options_, "int=10"));
+ ASSERT_OK(copy->ConfigureFromString(config_options_, "int=20"));
+ std::string bvalue, cvalue, mismatch;
+ ASSERT_OK(base->GetOption(config_options_, "int", &bvalue));
+ ASSERT_OK(copy->GetOption(config_options_, "int", &cvalue));
+ ASSERT_EQ(bvalue, "10");
+ ASSERT_EQ(cvalue, "20");
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_FALSE(copy->AreEquivalent(config_options_, base.get(), &mismatch));
+}
+
+TEST_F(ConfigurableTest, NullOptionMapTest) {
+ std::unique_ptr<Configurable> base;
+ std::unordered_set<std::string> names;
+ std::string str;
+
+ base.reset(
+ SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, nullptr));
+ ASSERT_NOK(base->ConfigureFromString(config_options_, "int=10"));
+ ASSERT_NOK(base->ConfigureFromString(config_options_, "int=20"));
+ ASSERT_NOK(base->ConfigureOption(config_options_, "int", "20"));
+ ASSERT_NOK(base->GetOption(config_options_, "int", &str));
+ ASSERT_NE(base->GetOptions<TestOptions>("c"), nullptr);
+ ASSERT_OK(base->GetOptionNames(config_options_, &names));
+ ASSERT_EQ(names.size(), 0UL);
+ ASSERT_OK(base->PrepareOptions(config_options_));
+ ASSERT_OK(base->ValidateOptions(DBOptions(), ColumnFamilyOptions()));
+ std::unique_ptr<Configurable> copy;
+ copy.reset(
+ SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode, nullptr));
+ ASSERT_OK(base->GetOptionString(config_options_, &str));
+ ASSERT_OK(copy->ConfigureFromString(config_options_, str));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &str));
+}
+#endif
+
+static std::unordered_map<std::string, ConfigTestFactoryFunc> TestFactories = {
+ {"Simple", []() { return SimpleConfigurable::Create("simple"); }},
+ {"Struct", []() { return SimpleStructFactory(); }},
+ {"Unique",
+ []() {
+ return SimpleConfigurable::Create(
+ "simple", TestConfigMode::kSimpleMode | TestConfigMode::kUniqueMode);
+ }},
+ {"Shared",
+ []() {
+ return SimpleConfigurable::Create(
+ "simple", TestConfigMode::kSimpleMode | TestConfigMode::kSharedMode);
+ }},
+ {"Nested",
+ []() {
+ return SimpleConfigurable::Create(
+ "simple", TestConfigMode::kSimpleMode | TestConfigMode::kNestedMode);
+ }},
+ {"Mutable",
+ []() {
+ return SimpleConfigurable::Create("simple",
+ TestConfigMode::kMutableMode |
+ TestConfigMode::kSimpleMode |
+ TestConfigMode::kNestedMode);
+ }},
+ {"ThreeDeep",
+ []() {
+ Configurable* simple = SimpleConfigurable::Create(
+ "Simple",
+ TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode);
+ auto* unique =
+ simple->GetOptions<std::unique_ptr<Configurable>>("SimpleUnique");
+ unique->reset(SimpleConfigurable::Create(
+ "Child",
+ TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode));
+ unique = unique->get()->GetOptions<std::unique_ptr<Configurable>>(
+ "ChildUnique");
+ unique->reset(
+ SimpleConfigurable::Create("Child", TestConfigMode::kDefaultMode));
+ return simple;
+ }},
+ {"DBOptions",
+ []() {
+ auto config = DBOptionsAsConfigurable(DBOptions());
+ return config.release();
+ }},
+ {"CFOptions",
+ []() {
+ auto config = CFOptionsAsConfigurable(ColumnFamilyOptions());
+ return config.release();
+ }},
+ {"BlockBased", []() { return NewBlockBasedTableFactory(); }},
+};
+
+class ConfigurableParamTest : public ConfigurableTest,
+ virtual public ::testing::WithParamInterface<
+ std::pair<std::string, std::string>> {
+ public:
+ ConfigurableParamTest() {
+ type_ = GetParam().first;
+ configuration_ = GetParam().second;
+ assert(TestFactories.find(type_) != TestFactories.end());
+ object_.reset(CreateConfigurable());
+ }
+
+ Configurable* CreateConfigurable() {
+ const auto& iter = TestFactories.find(type_);
+ return (iter->second)();
+ }
+
+ void TestConfigureOptions(const ConfigOptions& opts);
+ std::string type_;
+ std::string configuration_;
+ std::unique_ptr<Configurable> object_;
+};
+
+void ConfigurableParamTest::TestConfigureOptions(
+ const ConfigOptions& config_options) {
+ std::unique_ptr<Configurable> base, copy;
+ std::unordered_set<std::string> names;
+ std::string opt_str, mismatch;
+
+ base.reset(CreateConfigurable());
+ copy.reset(CreateConfigurable());
+
+ ASSERT_OK(base->ConfigureFromString(config_options, configuration_));
+ ASSERT_OK(base->GetOptionString(config_options, &opt_str));
+ ASSERT_OK(copy->ConfigureFromString(config_options, opt_str));
+ ASSERT_OK(copy->GetOptionString(config_options, &opt_str));
+ ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch));
+
+ copy.reset(CreateConfigurable());
+ ASSERT_OK(base->GetOptionNames(config_options, &names));
+ std::unordered_map<std::string, std::string> unused;
+ bool found_one = false;
+ for (auto name : names) {
+ std::string value;
+ Status s = base->GetOption(config_options, name, &value);
+ if (s.ok()) {
+ s = copy->ConfigureOption(config_options, name, value);
+ if (s.ok() || s.IsNotSupported()) {
+ found_one = true;
+ } else {
+ unused[name] = value;
+ }
+ } else {
+ ASSERT_TRUE(s.IsNotSupported());
+ }
+ }
+ ASSERT_TRUE(found_one || names.empty());
+ while (found_one && !unused.empty()) {
+ found_one = false;
+ for (auto iter = unused.begin(); iter != unused.end();) {
+ if (copy->ConfigureOption(config_options, iter->first, iter->second)
+ .ok()) {
+ found_one = true;
+ iter = unused.erase(iter);
+ } else {
+ ++iter;
+ }
+ }
+ }
+ ASSERT_EQ(0, unused.size());
+ ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch));
+}
+
+TEST_P(ConfigurableParamTest, GetDefaultOptionsTest) {
+ TestConfigureOptions(config_options_);
+}
+
+TEST_P(ConfigurableParamTest, ConfigureFromPropsTest) {
+ std::string opt_str, mismatch;
+ std::unordered_set<std::string> names;
+ std::unique_ptr<Configurable> copy(CreateConfigurable());
+
+ ASSERT_OK(object_->ConfigureFromString(config_options_, configuration_));
+ config_options_.delimiter = "\n";
+ ASSERT_OK(object_->GetOptionString(config_options_, &opt_str));
+ std::istringstream iss(opt_str);
+ std::unordered_map<std::string, std::string> copy_map;
+ std::string line;
+ for (int line_num = 0; std::getline(iss, line); line_num++) {
+ std::string name;
+ std::string value;
+ ASSERT_OK(
+ RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num));
+ copy_map[name] = value;
+ }
+ ASSERT_OK(copy->ConfigureFromMap(config_options_, copy_map));
+ ASSERT_TRUE(object_->AreEquivalent(config_options_, copy.get(), &mismatch));
+}
+
+INSTANTIATE_TEST_CASE_P(
+ ParamTest, ConfigurableParamTest,
+ testing::Values(
+ std::pair<std::string, std::string>("Simple",
+ "int=42;bool=true;string=s"),
+ std::pair<std::string, std::string>(
+ "Mutable", "int=42;unique={int=33;string=unique}"),
+ std::pair<std::string, std::string>(
+ "Struct", "struct={int=33;bool=true;string=s;}"),
+ std::pair<std::string, std::string>("Shared",
+ "int=33;bool=true;string=outer;"
+ "shared={int=42;string=shared}"),
+ std::pair<std::string, std::string>("Unique",
+ "int=33;bool=true;string=outer;"
+ "unique={int=42;string=unique}"),
+ std::pair<std::string, std::string>("Nested",
+ "int=11;bool=true;string=outer;"
+ "pointer={int=22;string=pointer};"
+ "unique={int=33;string=unique};"
+ "shared={int=44;string=shared}"),
+ std::pair<std::string, std::string>("ThreeDeep",
+ "int=11;bool=true;string=outer;"
+ "unique={int=22;string=inner;"
+ "unique={int=33;string=unique}};"),
+ std::pair<std::string, std::string>("DBOptions",
+ "max_background_jobs=100;"
+ "max_open_files=200;"),
+ std::pair<std::string, std::string>("CFOptions",
+ "table_factory=BlockBasedTable;"
+ "disable_auto_compactions=true;"),
+ std::pair<std::string, std::string>("BlockBased",
+ "block_size=1024;"
+ "no_block_cache=true;")));
+#endif // ROCKSDB_LITE
+
+} // namespace test
+} // namespace ROCKSDB_NAMESPACE
+int main(int argc, char** argv) {
+ ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+ ::testing::InitGoogleTest(&argc, argv);
+#ifdef GFLAGS
+ ParseCommandLineFlags(&argc, &argv, true);
+#endif // GFLAGS
+ return RUN_ALL_TESTS();
+}
diff --git a/src/rocksdb/options/configurable_test.h b/src/rocksdb/options/configurable_test.h
new file mode 100644
index 000000000..cf9d06678
--- /dev/null
+++ b/src/rocksdb/options/configurable_test.h
@@ -0,0 +1,126 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <algorithm>
+#include <memory>
+#include <unordered_map>
+
+#include "options/configurable_helper.h"
+#include "rocksdb/configurable.h"
+#include "rocksdb/utilities/options_type.h"
+
+namespace ROCKSDB_NAMESPACE {
+struct ColumnFamilyOptions;
+struct DBOptions;
+
+namespace test {
+enum TestEnum { kTestA, kTestB };
+
+static const std::unordered_map<std::string, int> test_enum_map = {
+ {"A", TestEnum::kTestA},
+ {"B", TestEnum::kTestB},
+};
+
+struct TestOptions {
+ int i = 0;
+ bool b = false;
+ bool d = true;
+ TestEnum e = TestEnum::kTestA;
+ std::string s = "";
+ std::string u = "";
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> simple_option_info = {
+#ifndef ROCKSDB_LITE
+ {"int",
+ {offsetof(struct TestOptions, i), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"bool",
+ {offsetof(struct TestOptions, b), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"string",
+ {offsetof(struct TestOptions, s), OptionType::kString,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+#endif // ROCKSDB_LITE
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> enum_option_info = {
+#ifndef ROCKSDB_LITE
+ {"enum",
+ OptionTypeInfo::Enum(offsetof(struct TestOptions, e), &test_enum_map)}
+#endif
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> unique_option_info = {
+#ifndef ROCKSDB_LITE
+ {"unique",
+ {0, OptionType::kConfigurable, OptionVerificationType::kNormal,
+ (OptionTypeFlags::kUnique | OptionTypeFlags::kMutable)}},
+#endif // ROCKSDB_LITE
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> shared_option_info = {
+#ifndef ROCKSDB_LITE
+ {"shared",
+ {0, OptionType::kConfigurable, OptionVerificationType::kNormal,
+ (OptionTypeFlags::kShared)}},
+#endif // ROCKSDB_LITE
+};
+static std::unordered_map<std::string, OptionTypeInfo> pointer_option_info = {
+#ifndef ROCKSDB_LITE
+ {"pointer",
+ {0, OptionType::kConfigurable, OptionVerificationType::kNormal,
+ OptionTypeFlags::kRawPointer}},
+#endif // ROCKSDB_LITE
+};
+
+enum TestConfigMode {
+ kEmptyMode = 0x0, // Don't register anything
+ kMutableMode = 0x01, // Configuration is mutable
+ kSimpleMode = 0x02, // Use the simple options
+ kEnumMode = 0x04, // Use the enum options
+ kDefaultMode = kSimpleMode, // Use no inner nested configurations
+ kSharedMode = 0x10, // Use shared configuration
+ kUniqueMode = 0x20, // Use unique configuration
+ kRawPtrMode = 0x40, // Use pointer configuration
+ kNestedMode = (kSharedMode | kUniqueMode | kRawPtrMode),
+ kAllOptMode = (kNestedMode | kEnumMode | kSimpleMode),
+};
+
+template <typename T>
+class TestConfigurable : public Configurable {
+ protected:
+ std::string name_;
+ std::string prefix_;
+ TestOptions options_;
+
+ public:
+ std::unique_ptr<T> unique_;
+ std::shared_ptr<T> shared_;
+ T* pointer_;
+
+ TestConfigurable(const std::string& name, int mode,
+ const std::unordered_map<std::string, OptionTypeInfo>* map =
+ &simple_option_info)
+ : name_(name), pointer_(nullptr) {
+ prefix_ = "test." + name + ".";
+ if ((mode & TestConfigMode::kSimpleMode) != 0) {
+ RegisterOptions(name_, &options_, map);
+ }
+ if ((mode & TestConfigMode::kEnumMode) != 0) {
+ RegisterOptions(name_ + "Enum", &options_, &enum_option_info);
+ }
+ }
+
+ ~TestConfigurable() override { delete pointer_; }
+};
+
+} // namespace test
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/customizable.cc b/src/rocksdb/options/customizable.cc
new file mode 100644
index 000000000..cd39550e5
--- /dev/null
+++ b/src/rocksdb/options/customizable.cc
@@ -0,0 +1,139 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "rocksdb/customizable.h"
+
+#include <sstream>
+
+#include "options/options_helper.h"
+#include "port/port.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/status.h"
+#include "rocksdb/utilities/options_type.h"
+#include "util/string_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+std::string Customizable::GetOptionName(const std::string& long_name) const {
+ const std::string& name = Name();
+ size_t name_len = name.size();
+ if (long_name.size() > name_len + 1 &&
+ long_name.compare(0, name_len, name) == 0 &&
+ long_name.at(name_len) == '.') {
+ return long_name.substr(name_len + 1);
+ } else {
+ return Configurable::GetOptionName(long_name);
+ }
+}
+
+std::string Customizable::GenerateIndividualId() const {
+ std::ostringstream ostr;
+ ostr << Name() << "@" << static_cast<const void*>(this) << "#"
+ << port::GetProcessID();
+ return ostr.str();
+}
+
+#ifndef ROCKSDB_LITE
+Status Customizable::GetOption(const ConfigOptions& config_options,
+ const std::string& opt_name,
+ std::string* value) const {
+ if (opt_name == OptionTypeInfo::kIdPropName()) {
+ *value = GetId();
+ return Status::OK();
+ } else {
+ return Configurable::GetOption(config_options, opt_name, value);
+ }
+}
+
+std::string Customizable::SerializeOptions(const ConfigOptions& config_options,
+ const std::string& prefix) const {
+ std::string result;
+ std::string parent;
+ std::string id = GetId();
+ if (!config_options.IsShallow() && !id.empty()) {
+ parent = Configurable::SerializeOptions(config_options, "");
+ }
+ if (parent.empty()) {
+ result = id;
+ } else {
+ result.append(prefix);
+ result.append(OptionTypeInfo::kIdPropName());
+ result.append("=");
+ result.append(id);
+ result.append(config_options.delimiter);
+ result.append(parent);
+ }
+ return result;
+}
+
+#endif // ROCKSDB_LITE
+
+bool Customizable::AreEquivalent(const ConfigOptions& config_options,
+ const Configurable* other,
+ std::string* mismatch) const {
+ if (config_options.sanity_level > ConfigOptions::kSanityLevelNone &&
+ this != other) {
+ const Customizable* custom = reinterpret_cast<const Customizable*>(other);
+ if (custom == nullptr) { // Cast failed
+ return false;
+ } else if (GetId() != custom->GetId()) {
+ *mismatch = OptionTypeInfo::kIdPropName();
+ return false;
+ } else if (config_options.sanity_level >
+ ConfigOptions::kSanityLevelLooselyCompatible) {
+ bool matches =
+ Configurable::AreEquivalent(config_options, other, mismatch);
+ return matches;
+ }
+ }
+ return true;
+}
+
+Status Customizable::GetOptionsMap(
+ const ConfigOptions& config_options, const Customizable* customizable,
+ const std::string& value, std::string* id,
+ std::unordered_map<std::string, std::string>* props) {
+ Status status;
+ if (value.empty() || value == kNullptrString) {
+ *id = "";
+ props->clear();
+ } else if (customizable != nullptr) {
+ status =
+ Configurable::GetOptionsMap(value, customizable->GetId(), id, props);
+#ifdef ROCKSDB_LITE
+ (void)config_options;
+#else
+ if (status.ok() && customizable->IsInstanceOf(*id)) {
+ // The new ID and the old ID match, so the objects are the same type.
+ // Try to get the existing options, ignoring any errors
+ ConfigOptions embedded = config_options;
+ embedded.delimiter = ";";
+ std::string curr_opts;
+ if (customizable->GetOptionString(embedded, &curr_opts).ok()) {
+ std::unordered_map<std::string, std::string> curr_props;
+ if (StringToMap(curr_opts, &curr_props).ok()) {
+ props->insert(curr_props.begin(), curr_props.end());
+ }
+ }
+ }
+#endif // ROCKSDB_LITE
+ } else {
+ status = Configurable::GetOptionsMap(value, "", id, props);
+ }
+ return status;
+}
+
+Status Customizable::ConfigureNewObject(
+ const ConfigOptions& config_options, Customizable* object,
+ const std::unordered_map<std::string, std::string>& opt_map) {
+ Status status;
+ if (object != nullptr) {
+ status = object->ConfigureFromMap(config_options, opt_map);
+ } else if (!opt_map.empty()) {
+ status = Status::InvalidArgument("Cannot configure null object ");
+ }
+ return status;
+}
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/customizable_test.cc b/src/rocksdb/options/customizable_test.cc
new file mode 100644
index 000000000..9d3c86c62
--- /dev/null
+++ b/src/rocksdb/options/customizable_test.cc
@@ -0,0 +1,2255 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "rocksdb/customizable.h"
+
+#include <cctype>
+#include <cinttypes>
+#include <cstring>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "db/db_test_util.h"
+#include "memory/jemalloc_nodump_allocator.h"
+#include "memory/memkind_kmem_allocator.h"
+#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "port/stack_trace.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/env_encryption.h"
+#include "rocksdb/file_checksum.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/flush_block_policy.h"
+#include "rocksdb/memory_allocator.h"
+#include "rocksdb/secondary_cache.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/sst_partitioner.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/utilities/customizable_util.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_type.h"
+#include "table/block_based/filter_policy_internal.h"
+#include "table/block_based/flush_block_policy.h"
+#include "table/mock_table.h"
+#include "test_util/mock_time_env.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+#include "util/file_checksum_helper.h"
+#include "util/string_util.h"
+#include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h"
+#include "utilities/memory_allocators.h"
+#include "utilities/merge_operators/bytesxor.h"
+#include "utilities/merge_operators/sortlist.h"
+#include "utilities/merge_operators/string_append/stringappend.h"
+#include "utilities/merge_operators/string_append/stringappend2.h"
+
+#ifndef GFLAGS
+bool FLAGS_enable_print = false;
+#else
+#include "util/gflags_compat.h"
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+DEFINE_bool(enable_print, false, "Print options generated to console.");
+#endif // GFLAGS
+
+namespace ROCKSDB_NAMESPACE {
+namespace {
+class StringLogger : public Logger {
+ public:
+ using Logger::Logv;
+ void Logv(const char* format, va_list ap) override {
+ char buffer[1000];
+ vsnprintf(buffer, sizeof(buffer), format, ap);
+ string_.append(buffer);
+ }
+ const std::string& str() const { return string_; }
+ void clear() { string_.clear(); }
+
+ private:
+ std::string string_;
+};
+
+class TestCustomizable : public Customizable {
+ public:
+ TestCustomizable(const std::string& name) : name_(name) {}
+ // Method to allow CheckedCast to work for this class
+ static const char* kClassName() {
+ return "TestCustomizable";
+ }
+
+ const char* Name() const override { return name_.c_str(); }
+ static const char* Type() { return "test.custom"; }
+#ifndef ROCKSDB_LITE
+ static Status CreateFromString(const ConfigOptions& opts,
+ const std::string& value,
+ std::unique_ptr<TestCustomizable>* result);
+ static Status CreateFromString(const ConfigOptions& opts,
+ const std::string& value,
+ std::shared_ptr<TestCustomizable>* result);
+ static Status CreateFromString(const ConfigOptions& opts,
+ const std::string& value,
+ TestCustomizable** result);
+#endif // ROCKSDB_LITE
+ bool IsInstanceOf(const std::string& name) const override {
+ if (name == kClassName()) {
+ return true;
+ } else {
+ return Customizable::IsInstanceOf(name);
+ }
+ }
+
+ protected:
+ const std::string name_;
+};
+
+struct AOptions {
+ static const char* kName() { return "A"; }
+ int i = 0;
+ bool b = false;
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> a_option_info = {
+#ifndef ROCKSDB_LITE
+ {"int",
+ {offsetof(struct AOptions, i), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"bool",
+ {offsetof(struct AOptions, b), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+#endif // ROCKSDB_LITE
+};
+
+class ACustomizable : public TestCustomizable {
+ public:
+ explicit ACustomizable(const std::string& id)
+ : TestCustomizable("A"), id_(id) {
+ RegisterOptions(&opts_, &a_option_info);
+ }
+ std::string GetId() const override { return id_; }
+ static const char* kClassName() { return "A"; }
+
+ private:
+ AOptions opts_;
+ const std::string id_;
+};
+
+struct BOptions {
+ std::string s;
+ bool b = false;
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> b_option_info = {
+#ifndef ROCKSDB_LITE
+ {"string",
+ {offsetof(struct BOptions, s), OptionType::kString,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"bool",
+ {offsetof(struct BOptions, b), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+#endif // ROCKSDB_LITE
+};
+
+class BCustomizable : public TestCustomizable {
+ private:
+ public:
+ explicit BCustomizable(const std::string& name) : TestCustomizable(name) {
+ RegisterOptions(name, &opts_, &b_option_info);
+ }
+ static const char* kClassName() { return "B"; }
+
+ private:
+ BOptions opts_;
+};
+
+#ifndef ROCKSDB_LITE
+static bool LoadSharedB(const std::string& id,
+ std::shared_ptr<TestCustomizable>* result) {
+ if (id == "B") {
+ result->reset(new BCustomizable(id));
+ return true;
+ } else if (id.empty()) {
+ result->reset();
+ return true;
+ } else {
+ return false;
+ }
+}
+
+static int A_count = 0;
+static int RegisterCustomTestObjects(ObjectLibrary& library,
+ const std::string& /*arg*/) {
+ library.AddFactory<TestCustomizable>(
+ ObjectLibrary::PatternEntry("A", true).AddSeparator("_"),
+ [](const std::string& name, std::unique_ptr<TestCustomizable>* guard,
+ std::string* /* msg */) {
+ guard->reset(new ACustomizable(name));
+ A_count++;
+ return guard->get();
+ });
+
+ library.AddFactory<TestCustomizable>(
+ "S", [](const std::string& name,
+ std::unique_ptr<TestCustomizable>* /* guard */,
+ std::string* /* msg */) { return new BCustomizable(name); });
+ size_t num_types;
+ return static_cast<int>(library.GetFactoryCount(&num_types));
+}
+#endif // ROCKSDB_LITE
+
+struct SimpleOptions {
+ static const char* kName() { return "simple"; }
+ bool b = true;
+ std::unique_ptr<TestCustomizable> cu;
+ std::shared_ptr<TestCustomizable> cs;
+ TestCustomizable* cp = nullptr;
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> simple_option_info = {
+#ifndef ROCKSDB_LITE
+ {"bool",
+ {offsetof(struct SimpleOptions, b), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"unique",
+ OptionTypeInfo::AsCustomUniquePtr<TestCustomizable>(
+ offsetof(struct SimpleOptions, cu), OptionVerificationType::kNormal,
+ OptionTypeFlags::kAllowNull)},
+ {"shared",
+ OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
+ offsetof(struct SimpleOptions, cs), OptionVerificationType::kNormal,
+ OptionTypeFlags::kAllowNull)},
+ {"pointer",
+ OptionTypeInfo::AsCustomRawPtr<TestCustomizable>(
+ offsetof(struct SimpleOptions, cp), OptionVerificationType::kNormal,
+ OptionTypeFlags::kAllowNull)},
+#endif // ROCKSDB_LITE
+};
+
+class SimpleConfigurable : public Configurable {
+ private:
+ SimpleOptions simple_;
+
+ public:
+ SimpleConfigurable() { RegisterOptions(&simple_, &simple_option_info); }
+
+ explicit SimpleConfigurable(
+ const std::unordered_map<std::string, OptionTypeInfo>* map) {
+ RegisterOptions(&simple_, map);
+ }
+};
+
+#ifndef ROCKSDB_LITE
+static void GetMapFromProperties(
+ const std::string& props,
+ std::unordered_map<std::string, std::string>* map) {
+ std::istringstream iss(props);
+ std::unordered_map<std::string, std::string> copy_map;
+ std::string line;
+ map->clear();
+ for (int line_num = 0; std::getline(iss, line); line_num++) {
+ std::string name;
+ std::string value;
+ ASSERT_OK(
+ RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num));
+ (*map)[name] = value;
+ }
+}
+#endif // ROCKSDB_LITE
+} // namespace
+
+#ifndef ROCKSDB_LITE
+Status TestCustomizable::CreateFromString(
+ const ConfigOptions& config_options, const std::string& value,
+ std::shared_ptr<TestCustomizable>* result) {
+ return LoadSharedObject<TestCustomizable>(config_options, value, LoadSharedB,
+ result);
+}
+
+Status TestCustomizable::CreateFromString(
+ const ConfigOptions& config_options, const std::string& value,
+ std::unique_ptr<TestCustomizable>* result) {
+ return LoadUniqueObject<TestCustomizable>(
+ config_options, value,
+ [](const std::string& id, std::unique_ptr<TestCustomizable>* u) {
+ if (id == "B") {
+ u->reset(new BCustomizable(id));
+ return true;
+ } else if (id.empty()) {
+ u->reset();
+ return true;
+ } else {
+ return false;
+ }
+ },
+ result);
+}
+
+Status TestCustomizable::CreateFromString(const ConfigOptions& config_options,
+ const std::string& value,
+ TestCustomizable** result) {
+ return LoadStaticObject<TestCustomizable>(
+ config_options, value,
+ [](const std::string& id, TestCustomizable** ptr) {
+ if (id == "B") {
+ *ptr = new BCustomizable(id);
+ return true;
+ } else if (id.empty()) {
+ *ptr = nullptr;
+ return true;
+ } else {
+ return false;
+ }
+ },
+ result);
+}
+#endif // ROCKSDB_LITE
+
+class CustomizableTest : public testing::Test {
+ public:
+ CustomizableTest() {
+ config_options_.invoke_prepare_options = false;
+#ifndef ROCKSDB_LITE
+ // GetOptionsFromMap is not supported in ROCKSDB_LITE
+ config_options_.registry->AddLibrary("CustomizableTest",
+ RegisterCustomTestObjects, "");
+#endif // ROCKSDB_LITE
+ }
+
+ ConfigOptions config_options_;
+};
+
+#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
+// Tests that a Customizable can be created by:
+// - a simple name
+// - a XXX.id option
+// - a property with a name
+TEST_F(CustomizableTest, CreateByNameTest) {
+ ObjectLibrary::Default()->AddFactory<TestCustomizable>(
+ ObjectLibrary::PatternEntry("TEST", false).AddSeparator("_"),
+ [](const std::string& name, std::unique_ptr<TestCustomizable>* guard,
+ std::string* /* msg */) {
+ guard->reset(new TestCustomizable(name));
+ return guard->get();
+ });
+ std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
+ SimpleOptions* simple = configurable->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_OK(
+ configurable->ConfigureFromString(config_options_, "unique={id=TEST_1}"));
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(simple->cu->GetId(), "TEST_1");
+ ASSERT_OK(
+ configurable->ConfigureFromString(config_options_, "unique.id=TEST_2"));
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(simple->cu->GetId(), "TEST_2");
+ ASSERT_OK(
+ configurable->ConfigureFromString(config_options_, "unique=TEST_3"));
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(simple->cu->GetId(), "TEST_3");
+}
+
+TEST_F(CustomizableTest, ToStringTest) {
+ std::unique_ptr<TestCustomizable> custom(new TestCustomizable("test"));
+ ASSERT_EQ(custom->ToString(config_options_), "test");
+}
+
+TEST_F(CustomizableTest, SimpleConfigureTest) {
+ std::unordered_map<std::string, std::string> opt_map = {
+ {"unique", "id=A;int=1;bool=true"},
+ {"shared", "id=B;string=s"},
+ };
+ std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
+ ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map));
+ SimpleOptions* simple = configurable->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(simple->cu->GetId(), "A");
+ std::string opt_str;
+ std::string mismatch;
+ ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str));
+ std::unique_ptr<Configurable> copy(new SimpleConfigurable());
+ ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
+ ASSERT_TRUE(
+ configurable->AreEquivalent(config_options_, copy.get(), &mismatch));
+}
+
+TEST_F(CustomizableTest, ConfigureFromPropsTest) {
+ std::unordered_map<std::string, std::string> opt_map = {
+ {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"},
+ {"shared.id", "B"}, {"shared.B.string", "s"},
+ };
+ std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
+ ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map));
+ SimpleOptions* simple = configurable->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(simple->cu->GetId(), "A");
+ std::string opt_str;
+ std::string mismatch;
+ config_options_.delimiter = "\n";
+ std::unordered_map<std::string, std::string> props;
+ ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str));
+ GetMapFromProperties(opt_str, &props);
+ std::unique_ptr<Configurable> copy(new SimpleConfigurable());
+ ASSERT_OK(copy->ConfigureFromMap(config_options_, props));
+ ASSERT_TRUE(
+ configurable->AreEquivalent(config_options_, copy.get(), &mismatch));
+}
+
+TEST_F(CustomizableTest, ConfigureFromShortTest) {
+ std::unordered_map<std::string, std::string> opt_map = {
+ {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"},
+ {"shared.id", "B"}, {"shared.B.string", "s"},
+ };
+ std::unique_ptr<Configurable> configurable(new SimpleConfigurable());
+ ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map));
+ SimpleOptions* simple = configurable->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(simple->cu->GetId(), "A");
+}
+
+TEST_F(CustomizableTest, AreEquivalentOptionsTest) {
+ std::unordered_map<std::string, std::string> opt_map = {
+ {"unique", "id=A;int=1;bool=true"},
+ {"shared", "id=A;int=1;bool=true"},
+ };
+ std::string mismatch;
+ ConfigOptions config_options = config_options_;
+ std::unique_ptr<Configurable> c1(new SimpleConfigurable());
+ std::unique_ptr<Configurable> c2(new SimpleConfigurable());
+ ASSERT_OK(c1->ConfigureFromMap(config_options, opt_map));
+ ASSERT_OK(c2->ConfigureFromMap(config_options, opt_map));
+ ASSERT_TRUE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
+ SimpleOptions* simple = c1->GetOptions<SimpleOptions>();
+ ASSERT_TRUE(
+ simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch));
+ ASSERT_OK(simple->cu->ConfigureOption(config_options, "int", "2"));
+ ASSERT_FALSE(
+ simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch));
+ ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
+ ConfigOptions loosely = config_options;
+ loosely.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
+ ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch));
+ ASSERT_TRUE(simple->cu->AreEquivalent(loosely, simple->cs.get(), &mismatch));
+
+ ASSERT_OK(c1->ConfigureOption(config_options, "shared", "id=B;string=3"));
+ ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch));
+ ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
+ ASSERT_FALSE(simple->cs->AreEquivalent(loosely, simple->cu.get(), &mismatch));
+ simple->cs.reset();
+ ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch));
+ ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch));
+}
+
+// Tests that we can initialize a customizable from its options
+TEST_F(CustomizableTest, ConfigureStandaloneCustomTest) {
+ std::unique_ptr<TestCustomizable> base, copy;
+ const auto& registry = config_options_.registry;
+ ASSERT_OK(registry->NewUniqueObject<TestCustomizable>("A", &base));
+ ASSERT_OK(registry->NewUniqueObject<TestCustomizable>("A", &copy));
+ ASSERT_OK(base->ConfigureFromString(config_options_, "int=33;bool=true"));
+ std::string opt_str;
+ std::string mismatch;
+ ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
+ ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+}
+
+// Tests that we fail appropriately if the pattern is not registered
+TEST_F(CustomizableTest, BadNameTest) {
+ config_options_.ignore_unsupported_options = false;
+ std::unique_ptr<Configurable> c1(new SimpleConfigurable());
+ ASSERT_NOK(
+ c1->ConfigureFromString(config_options_, "unique.shared.id=bad name"));
+ config_options_.ignore_unsupported_options = true;
+ ASSERT_OK(
+ c1->ConfigureFromString(config_options_, "unique.shared.id=bad name"));
+}
+
+// Tests that we fail appropriately if a bad option is passed to the underlying
+// configurable
+TEST_F(CustomizableTest, BadOptionTest) {
+ std::unique_ptr<Configurable> c1(new SimpleConfigurable());
+ ConfigOptions ignore = config_options_;
+ ignore.ignore_unknown_options = true;
+
+ ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.int=11"));
+ ASSERT_NOK(c1->ConfigureFromString(config_options_, "shared={id=B;int=1}"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "shared={id=A;string=s}"));
+ ASSERT_NOK(c1->ConfigureFromString(config_options_, "B.int=11"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "B.int=11"));
+ ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.string=s"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "A.string=s"));
+ // Test as detached
+ ASSERT_NOK(
+ c1->ConfigureFromString(config_options_, "shared.id=A;A.string=b}"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=A;A.string=s}"));
+}
+
+TEST_F(CustomizableTest, FailingFactoryTest) {
+ std::shared_ptr<ObjectRegistry> registry = ObjectRegistry::NewInstance();
+ std::unique_ptr<Configurable> c1(new SimpleConfigurable());
+ ConfigOptions ignore = config_options_;
+
+ Status s;
+ ignore.registry->AddLibrary("failing")->AddFactory<TestCustomizable>(
+ "failing",
+ [](const std::string& /*uri*/,
+ std::unique_ptr<TestCustomizable>* /*guard */, std::string* errmsg) {
+ *errmsg = "Bad Factory";
+ return nullptr;
+ });
+
+ // If we are ignoring unknown and unsupported options, will see
+ // different errors for failing versus missing
+ ignore.ignore_unknown_options = false;
+ ignore.ignore_unsupported_options = false;
+ s = c1->ConfigureFromString(ignore, "shared.id=failing");
+ ASSERT_TRUE(s.IsInvalidArgument());
+ s = c1->ConfigureFromString(ignore, "unique.id=failing");
+ ASSERT_TRUE(s.IsInvalidArgument());
+ s = c1->ConfigureFromString(ignore, "shared.id=missing");
+ ASSERT_TRUE(s.IsNotSupported());
+ s = c1->ConfigureFromString(ignore, "unique.id=missing");
+ ASSERT_TRUE(s.IsNotSupported());
+
+ // If we are ignoring unsupported options, will see
+ // errors for failing but not missing
+ ignore.ignore_unknown_options = false;
+ ignore.ignore_unsupported_options = true;
+ s = c1->ConfigureFromString(ignore, "shared.id=failing");
+ ASSERT_TRUE(s.IsInvalidArgument());
+ s = c1->ConfigureFromString(ignore, "unique.id=failing");
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=missing"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "unique.id=missing"));
+
+ // If we are ignoring unknown options, will see no errors
+ // for failing or missing
+ ignore.ignore_unknown_options = true;
+ ignore.ignore_unsupported_options = false;
+ ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=failing"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "unique.id=failing"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=missing"));
+ ASSERT_OK(c1->ConfigureFromString(ignore, "unique.id=missing"));
+}
+
+// Tests that different IDs lead to different objects
+TEST_F(CustomizableTest, UniqueIdTest) {
+ std::unique_ptr<Configurable> base(new SimpleConfigurable());
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "unique={id=A_1;int=1;bool=true}"));
+ SimpleOptions* simple = base->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(simple->cu->GetId(), std::string("A_1"));
+ std::string opt_str;
+ std::string mismatch;
+ ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
+ std::unique_ptr<Configurable> copy(new SimpleConfigurable());
+ ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
+ ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "unique={id=A_2;int=1;bool=true}"));
+ ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
+ ASSERT_EQ(simple->cu->GetId(), std::string("A_2"));
+}
+
+TEST_F(CustomizableTest, IsInstanceOfTest) {
+ std::shared_ptr<TestCustomizable> tc = std::make_shared<ACustomizable>("A_1");
+
+ ASSERT_EQ(tc->GetId(), std::string("A_1"));
+ ASSERT_TRUE(tc->IsInstanceOf("A"));
+ ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable"));
+ ASSERT_FALSE(tc->IsInstanceOf("B"));
+ ASSERT_FALSE(tc->IsInstanceOf("A_1"));
+ ASSERT_EQ(tc->CheckedCast<ACustomizable>(), tc.get());
+ ASSERT_EQ(tc->CheckedCast<TestCustomizable>(), tc.get());
+ ASSERT_EQ(tc->CheckedCast<BCustomizable>(), nullptr);
+
+ tc.reset(new BCustomizable("B"));
+ ASSERT_TRUE(tc->IsInstanceOf("B"));
+ ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable"));
+ ASSERT_FALSE(tc->IsInstanceOf("A"));
+ ASSERT_EQ(tc->CheckedCast<BCustomizable>(), tc.get());
+ ASSERT_EQ(tc->CheckedCast<TestCustomizable>(), tc.get());
+ ASSERT_EQ(tc->CheckedCast<ACustomizable>(), nullptr);
+}
+
+TEST_F(CustomizableTest, PrepareOptionsTest) {
+ static std::unordered_map<std::string, OptionTypeInfo> p_option_info = {
+#ifndef ROCKSDB_LITE
+ {"can_prepare",
+ {0, OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+#endif // ROCKSDB_LITE
+ };
+
+ class PrepareCustomizable : public TestCustomizable {
+ public:
+ bool can_prepare_ = true;
+
+ PrepareCustomizable() : TestCustomizable("P") {
+ RegisterOptions("Prepare", &can_prepare_, &p_option_info);
+ }
+
+ Status PrepareOptions(const ConfigOptions& opts) override {
+ if (!can_prepare_) {
+ return Status::InvalidArgument("Cannot Prepare");
+ } else {
+ return TestCustomizable::PrepareOptions(opts);
+ }
+ }
+ };
+
+ ObjectLibrary::Default()->AddFactory<TestCustomizable>(
+ "P",
+ [](const std::string& /*name*/, std::unique_ptr<TestCustomizable>* guard,
+ std::string* /* msg */) {
+ guard->reset(new PrepareCustomizable());
+ return guard->get();
+ });
+
+ std::unique_ptr<Configurable> base(new SimpleConfigurable());
+ ConfigOptions prepared(config_options_);
+ prepared.invoke_prepare_options = true;
+
+ ASSERT_OK(base->ConfigureFromString(
+ prepared, "unique=A_1; shared={id=B;string=s}; pointer.id=S"));
+ SimpleOptions* simple = base->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_NE(simple->cs, nullptr);
+ ASSERT_NE(simple->cp, nullptr);
+ delete simple->cp;
+ base.reset(new SimpleConfigurable());
+ ASSERT_OK(base->ConfigureFromString(
+ config_options_, "unique=A_1; shared={id=B;string=s}; pointer.id=S"));
+
+ simple = base->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_NE(simple->cs, nullptr);
+ ASSERT_NE(simple->cp, nullptr);
+
+ ASSERT_OK(base->PrepareOptions(config_options_));
+ delete simple->cp;
+ base.reset(new SimpleConfigurable());
+ simple = base->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+
+ ASSERT_NOK(
+ base->ConfigureFromString(prepared, "unique={id=P; can_prepare=false}"));
+ ASSERT_EQ(simple->cu, nullptr);
+
+ ASSERT_OK(
+ base->ConfigureFromString(prepared, "unique={id=P; can_prepare=true}"));
+ ASSERT_NE(simple->cu, nullptr);
+
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "unique={id=P; can_prepare=true}"));
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_OK(simple->cu->PrepareOptions(prepared));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "unique={id=P; can_prepare=false}"));
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_NOK(simple->cu->PrepareOptions(prepared));
+}
+
+namespace {
+static std::unordered_map<std::string, OptionTypeInfo> inner_option_info = {
+#ifndef ROCKSDB_LITE
+ {"inner",
+ OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kStringNameOnly)}
+#endif // ROCKSDB_LITE
+};
+
+struct InnerOptions {
+ static const char* kName() { return "InnerOptions"; }
+ std::shared_ptr<Customizable> inner;
+};
+
+class InnerCustomizable : public Customizable {
+ public:
+ explicit InnerCustomizable(const std::shared_ptr<Customizable>& w) {
+ iopts_.inner = w;
+ RegisterOptions(&iopts_, &inner_option_info);
+ }
+ static const char* kClassName() { return "Inner"; }
+ const char* Name() const override { return kClassName(); }
+
+ bool IsInstanceOf(const std::string& name) const override {
+ if (name == kClassName()) {
+ return true;
+ } else {
+ return Customizable::IsInstanceOf(name);
+ }
+ }
+
+ protected:
+ const Customizable* Inner() const override { return iopts_.inner.get(); }
+
+ private:
+ InnerOptions iopts_;
+};
+
+struct WrappedOptions1 {
+ static const char* kName() { return "WrappedOptions1"; }
+ int i = 42;
+};
+
+class WrappedCustomizable1 : public InnerCustomizable {
+ public:
+ explicit WrappedCustomizable1(const std::shared_ptr<Customizable>& w)
+ : InnerCustomizable(w) {
+ RegisterOptions(&wopts_, nullptr);
+ }
+ const char* Name() const override { return kClassName(); }
+ static const char* kClassName() { return "Wrapped1"; }
+
+ private:
+ WrappedOptions1 wopts_;
+};
+
+struct WrappedOptions2 {
+ static const char* kName() { return "WrappedOptions2"; }
+ std::string s = "42";
+};
+class WrappedCustomizable2 : public InnerCustomizable {
+ public:
+ explicit WrappedCustomizable2(const std::shared_ptr<Customizable>& w)
+ : InnerCustomizable(w) {}
+ const void* GetOptionsPtr(const std::string& name) const override {
+ if (name == WrappedOptions2::kName()) {
+ return &wopts_;
+ } else {
+ return InnerCustomizable::GetOptionsPtr(name);
+ }
+ }
+
+ const char* Name() const override { return kClassName(); }
+ static const char* kClassName() { return "Wrapped2"; }
+
+ private:
+ WrappedOptions2 wopts_;
+};
+} // namespace
+
+TEST_F(CustomizableTest, WrappedInnerTest) {
+ std::shared_ptr<TestCustomizable> ac =
+ std::make_shared<TestCustomizable>("A");
+
+ ASSERT_TRUE(ac->IsInstanceOf("A"));
+ ASSERT_TRUE(ac->IsInstanceOf("TestCustomizable"));
+ ASSERT_EQ(ac->CheckedCast<TestCustomizable>(), ac.get());
+ ASSERT_EQ(ac->CheckedCast<InnerCustomizable>(), nullptr);
+ ASSERT_EQ(ac->CheckedCast<WrappedCustomizable1>(), nullptr);
+ ASSERT_EQ(ac->CheckedCast<WrappedCustomizable2>(), nullptr);
+ std::shared_ptr<Customizable> wc1 =
+ std::make_shared<WrappedCustomizable1>(ac);
+
+ ASSERT_TRUE(wc1->IsInstanceOf(WrappedCustomizable1::kClassName()));
+ ASSERT_EQ(wc1->CheckedCast<WrappedCustomizable1>(), wc1.get());
+ ASSERT_EQ(wc1->CheckedCast<WrappedCustomizable2>(), nullptr);
+ ASSERT_EQ(wc1->CheckedCast<InnerCustomizable>(), wc1.get());
+ ASSERT_EQ(wc1->CheckedCast<TestCustomizable>(), ac.get());
+
+ std::shared_ptr<Customizable> wc2 =
+ std::make_shared<WrappedCustomizable2>(wc1);
+ ASSERT_TRUE(wc2->IsInstanceOf(WrappedCustomizable2::kClassName()));
+ ASSERT_EQ(wc2->CheckedCast<WrappedCustomizable2>(), wc2.get());
+ ASSERT_EQ(wc2->CheckedCast<WrappedCustomizable1>(), wc1.get());
+ ASSERT_EQ(wc2->CheckedCast<InnerCustomizable>(), wc2.get());
+ ASSERT_EQ(wc2->CheckedCast<TestCustomizable>(), ac.get());
+}
+
+TEST_F(CustomizableTest, CustomizableInnerTest) {
+ std::shared_ptr<Customizable> c =
+ std::make_shared<InnerCustomizable>(std::make_shared<ACustomizable>("a"));
+ std::shared_ptr<Customizable> wc1 = std::make_shared<WrappedCustomizable1>(c);
+ std::shared_ptr<Customizable> wc2 = std::make_shared<WrappedCustomizable2>(c);
+ auto inner = c->GetOptions<InnerOptions>();
+ ASSERT_NE(inner, nullptr);
+
+ auto aopts = c->GetOptions<AOptions>();
+ ASSERT_NE(aopts, nullptr);
+ ASSERT_EQ(aopts, wc1->GetOptions<AOptions>());
+ ASSERT_EQ(aopts, wc2->GetOptions<AOptions>());
+ auto w1opts = wc1->GetOptions<WrappedOptions1>();
+ ASSERT_NE(w1opts, nullptr);
+ ASSERT_EQ(c->GetOptions<WrappedOptions1>(), nullptr);
+ ASSERT_EQ(wc2->GetOptions<WrappedOptions1>(), nullptr);
+
+ auto w2opts = wc2->GetOptions<WrappedOptions2>();
+ ASSERT_NE(w2opts, nullptr);
+ ASSERT_EQ(c->GetOptions<WrappedOptions2>(), nullptr);
+ ASSERT_EQ(wc1->GetOptions<WrappedOptions2>(), nullptr);
+}
+
+TEST_F(CustomizableTest, CopyObjectTest) {
+ class CopyCustomizable : public Customizable {
+ public:
+ CopyCustomizable() : prepared_(0), validated_(0) {}
+ const char* Name() const override { return "CopyCustomizable"; }
+
+ Status PrepareOptions(const ConfigOptions& options) override {
+ prepared_++;
+ return Customizable::PrepareOptions(options);
+ }
+ Status ValidateOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts) const override {
+ validated_++;
+ return Customizable::ValidateOptions(db_opts, cf_opts);
+ }
+ int prepared_;
+ mutable int validated_;
+ };
+
+ CopyCustomizable c1;
+ ConfigOptions config_options;
+ Options options;
+
+ ASSERT_OK(c1.PrepareOptions(config_options));
+ ASSERT_OK(c1.ValidateOptions(options, options));
+ ASSERT_EQ(c1.prepared_, 1);
+ ASSERT_EQ(c1.validated_, 1);
+ CopyCustomizable c2 = c1;
+ ASSERT_OK(c1.PrepareOptions(config_options));
+ ASSERT_OK(c1.ValidateOptions(options, options));
+ ASSERT_EQ(c2.prepared_, 1);
+ ASSERT_EQ(c2.validated_, 1);
+ ASSERT_EQ(c1.prepared_, 2);
+ ASSERT_EQ(c1.validated_, 2);
+}
+
+TEST_F(CustomizableTest, TestStringDepth) {
+ ConfigOptions shallow = config_options_;
+ std::unique_ptr<Configurable> c(
+ new InnerCustomizable(std::make_shared<ACustomizable>("a")));
+ std::string opt_str;
+ shallow.depth = ConfigOptions::Depth::kDepthShallow;
+ ASSERT_OK(c->GetOptionString(shallow, &opt_str));
+ ASSERT_EQ(opt_str, "inner=a;");
+ shallow.depth = ConfigOptions::Depth::kDepthDetailed;
+ ASSERT_OK(c->GetOptionString(shallow, &opt_str));
+ ASSERT_NE(opt_str, "inner=a;");
+}
+
+// Tests that we only get a new customizable when it changes
+TEST_F(CustomizableTest, NewUniqueCustomizableTest) {
+ std::unique_ptr<Configurable> base(new SimpleConfigurable());
+ A_count = 0;
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "unique={id=A_1;int=1;bool=true}"));
+ SimpleOptions* simple = base->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_NE(simple->cu, nullptr);
+ ASSERT_EQ(A_count, 1); // Created one A
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "unique={id=A_1;int=1;bool=false}"));
+ ASSERT_EQ(A_count, 2); // Create another A_1
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique={id=}"));
+ ASSERT_EQ(simple->cu, nullptr);
+ ASSERT_EQ(A_count, 2);
+ ASSERT_OK(base->ConfigureFromString(config_options_,
+ "unique={id=A_2;int=1;bool=false}"));
+ ASSERT_EQ(A_count, 3); // Created another A
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id="));
+ ASSERT_EQ(simple->cu, nullptr);
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique=nullptr"));
+ ASSERT_EQ(simple->cu, nullptr);
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id=nullptr"));
+ ASSERT_EQ(simple->cu, nullptr);
+ ASSERT_EQ(A_count, 3);
+}
+
+TEST_F(CustomizableTest, NewEmptyUniqueTest) {
+ std::unique_ptr<Configurable> base(new SimpleConfigurable());
+ SimpleOptions* simple = base->GetOptions<SimpleOptions>();
+ ASSERT_EQ(simple->cu, nullptr);
+ simple->cu.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique={id=}"));
+ ASSERT_EQ(simple->cu, nullptr);
+ simple->cu.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique={id=nullptr}"));
+ ASSERT_EQ(simple->cu, nullptr);
+ simple->cu.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id="));
+ ASSERT_EQ(simple->cu, nullptr);
+ simple->cu.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique=nullptr"));
+ ASSERT_EQ(simple->cu, nullptr);
+ simple->cu.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "unique.id=nullptr"));
+ ASSERT_EQ(simple->cu, nullptr);
+}
+
+TEST_F(CustomizableTest, NewEmptySharedTest) {
+ std::unique_ptr<Configurable> base(new SimpleConfigurable());
+
+ SimpleOptions* simple = base->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_EQ(simple->cs, nullptr);
+ simple->cs.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "shared={id=}"));
+ ASSERT_NE(simple, nullptr);
+ ASSERT_EQ(simple->cs, nullptr);
+ simple->cs.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "shared={id=nullptr}"));
+ ASSERT_EQ(simple->cs, nullptr);
+ simple->cs.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "shared.id="));
+ ASSERT_EQ(simple->cs, nullptr);
+ simple->cs.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "shared.id=nullptr"));
+ ASSERT_EQ(simple->cs, nullptr);
+ simple->cs.reset(new BCustomizable("B"));
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "shared=nullptr"));
+ ASSERT_EQ(simple->cs, nullptr);
+}
+
+TEST_F(CustomizableTest, NewEmptyStaticTest) {
+ std::unique_ptr<Configurable> base(new SimpleConfigurable());
+ ASSERT_OK(base->ConfigureFromString(config_options_, "pointer={id=}"));
+ SimpleOptions* simple = base->GetOptions<SimpleOptions>();
+ ASSERT_NE(simple, nullptr);
+ ASSERT_EQ(simple->cp, nullptr);
+ ASSERT_OK(base->ConfigureFromString(config_options_, "pointer={id=nullptr}"));
+ ASSERT_EQ(simple->cp, nullptr);
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "pointer="));
+ ASSERT_EQ(simple->cp, nullptr);
+ ASSERT_OK(base->ConfigureFromString(config_options_, "pointer=nullptr"));
+ ASSERT_EQ(simple->cp, nullptr);
+
+ ASSERT_OK(base->ConfigureFromString(config_options_, "pointer.id="));
+ ASSERT_EQ(simple->cp, nullptr);
+ ASSERT_OK(base->ConfigureFromString(config_options_, "pointer.id=nullptr"));
+ ASSERT_EQ(simple->cp, nullptr);
+}
+
+namespace {
+#ifndef ROCKSDB_LITE
+static std::unordered_map<std::string, OptionTypeInfo> vector_option_info = {
+ {"vector",
+ OptionTypeInfo::Vector<std::shared_ptr<TestCustomizable>>(
+ 0, OptionVerificationType::kNormal,
+
+ OptionTypeFlags::kNone,
+
+ OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone))},
+};
+class VectorConfigurable : public SimpleConfigurable {
+ public:
+ VectorConfigurable() { RegisterOptions("vector", &cv, &vector_option_info); }
+ std::vector<std::shared_ptr<TestCustomizable>> cv;
+};
+} // namespace
+
+TEST_F(CustomizableTest, VectorConfigTest) {
+ VectorConfigurable orig, copy;
+ std::shared_ptr<TestCustomizable> c1, c2;
+ ASSERT_OK(TestCustomizable::CreateFromString(config_options_, "A", &c1));
+ ASSERT_OK(TestCustomizable::CreateFromString(config_options_, "B", &c2));
+ orig.cv.push_back(c1);
+ orig.cv.push_back(c2);
+ ASSERT_OK(orig.ConfigureFromString(config_options_, "unique=A2"));
+ std::string opt_str, mismatch;
+ ASSERT_OK(orig.GetOptionString(config_options_, &opt_str));
+ ASSERT_OK(copy.ConfigureFromString(config_options_, opt_str));
+ ASSERT_TRUE(orig.AreEquivalent(config_options_, &copy, &mismatch));
+}
+
+TEST_F(CustomizableTest, NoNameTest) {
+ // If Customizables are created without names, they are not
+ // part of the serialization (since they cannot be recreated)
+ VectorConfigurable orig, copy;
+ auto sopts = orig.GetOptions<SimpleOptions>();
+ auto copts = copy.GetOptions<SimpleOptions>();
+ sopts->cu.reset(new ACustomizable(""));
+ orig.cv.push_back(std::make_shared<ACustomizable>(""));
+ orig.cv.push_back(std::make_shared<ACustomizable>("A_1"));
+ std::string opt_str, mismatch;
+ ASSERT_OK(orig.GetOptionString(config_options_, &opt_str));
+ ASSERT_OK(copy.ConfigureFromString(config_options_, opt_str));
+ ASSERT_EQ(copy.cv.size(), 1U);
+ ASSERT_EQ(copy.cv[0]->GetId(), "A_1");
+ ASSERT_EQ(copts->cu, nullptr);
+}
+
+#endif // ROCKSDB_LITE
+
+TEST_F(CustomizableTest, IgnoreUnknownObjects) {
+ ConfigOptions ignore = config_options_;
+ std::shared_ptr<TestCustomizable> shared;
+ std::unique_ptr<TestCustomizable> unique;
+ TestCustomizable* pointer = nullptr;
+ ignore.ignore_unsupported_options = false;
+ ASSERT_NOK(
+ LoadSharedObject<TestCustomizable>(ignore, "Unknown", nullptr, &shared));
+ ASSERT_NOK(
+ LoadUniqueObject<TestCustomizable>(ignore, "Unknown", nullptr, &unique));
+ ASSERT_NOK(
+ LoadStaticObject<TestCustomizable>(ignore, "Unknown", nullptr, &pointer));
+ ASSERT_EQ(shared.get(), nullptr);
+ ASSERT_EQ(unique.get(), nullptr);
+ ASSERT_EQ(pointer, nullptr);
+ ignore.ignore_unsupported_options = true;
+ ASSERT_OK(
+ LoadSharedObject<TestCustomizable>(ignore, "Unknown", nullptr, &shared));
+ ASSERT_OK(
+ LoadUniqueObject<TestCustomizable>(ignore, "Unknown", nullptr, &unique));
+ ASSERT_OK(
+ LoadStaticObject<TestCustomizable>(ignore, "Unknown", nullptr, &pointer));
+ ASSERT_EQ(shared.get(), nullptr);
+ ASSERT_EQ(unique.get(), nullptr);
+ ASSERT_EQ(pointer, nullptr);
+ ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown", nullptr,
+ &shared));
+ ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown", nullptr,
+ &unique));
+ ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown", nullptr,
+ &pointer));
+ ASSERT_EQ(shared.get(), nullptr);
+ ASSERT_EQ(unique.get(), nullptr);
+ ASSERT_EQ(pointer, nullptr);
+ ASSERT_OK(LoadSharedObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
+ nullptr, &shared));
+ ASSERT_OK(LoadUniqueObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
+ nullptr, &unique));
+ ASSERT_OK(LoadStaticObject<TestCustomizable>(ignore, "id=Unknown;option=bad",
+ nullptr, &pointer));
+ ASSERT_EQ(shared.get(), nullptr);
+ ASSERT_EQ(unique.get(), nullptr);
+ ASSERT_EQ(pointer, nullptr);
+}
+
+TEST_F(CustomizableTest, FactoryFunctionTest) {
+ std::shared_ptr<TestCustomizable> shared;
+ std::unique_ptr<TestCustomizable> unique;
+ TestCustomizable* pointer = nullptr;
+ ConfigOptions ignore = config_options_;
+ ignore.ignore_unsupported_options = false;
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &shared));
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &unique));
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &pointer));
+ ASSERT_NE(shared.get(), nullptr);
+ ASSERT_NE(unique.get(), nullptr);
+ ASSERT_NE(pointer, nullptr);
+ delete pointer;
+ pointer = nullptr;
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "id=", &shared));
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "id=", &unique));
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "id=", &pointer));
+ ASSERT_EQ(shared.get(), nullptr);
+ ASSERT_EQ(unique.get(), nullptr);
+ ASSERT_EQ(pointer, nullptr);
+ ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &shared));
+ ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &unique));
+ ASSERT_NOK(
+ TestCustomizable::CreateFromString(ignore, "option=bad", &pointer));
+ ASSERT_EQ(pointer, nullptr);
+}
+
+TEST_F(CustomizableTest, URLFactoryTest) {
+ std::unique_ptr<TestCustomizable> unique;
+ config_options_.registry->AddLibrary("URL")->AddFactory<TestCustomizable>(
+ ObjectLibrary::PatternEntry("Z", false).AddSeparator(""),
+ [](const std::string& name, std::unique_ptr<TestCustomizable>* guard,
+ std::string* /* msg */) {
+ guard->reset(new TestCustomizable(name));
+ return guard->get();
+ });
+
+ ConfigOptions ignore = config_options_;
+ ignore.ignore_unsupported_options = false;
+ ignore.ignore_unsupported_options = false;
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "Z=1;x=y", &unique));
+ ASSERT_NE(unique, nullptr);
+ ASSERT_EQ(unique->GetId(), "Z=1;x=y");
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "Z;x=y", &unique));
+ ASSERT_NE(unique, nullptr);
+ ASSERT_EQ(unique->GetId(), "Z;x=y");
+ unique.reset();
+ ASSERT_OK(TestCustomizable::CreateFromString(ignore, "Z=1?x=y", &unique));
+ ASSERT_NE(unique, nullptr);
+ ASSERT_EQ(unique->GetId(), "Z=1?x=y");
+}
+
+TEST_F(CustomizableTest, MutableOptionsTest) {
+ static std::unordered_map<std::string, OptionTypeInfo> mutable_option_info = {
+ {"mutable",
+ OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kMutable)}};
+ static std::unordered_map<std::string, OptionTypeInfo> immutable_option_info =
+ {{"immutable",
+ OptionTypeInfo::AsCustomSharedPtr<TestCustomizable>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kAllowNull)}};
+
+ class MutableCustomizable : public Customizable {
+ private:
+ std::shared_ptr<TestCustomizable> mutable_;
+ std::shared_ptr<TestCustomizable> immutable_;
+
+ public:
+ MutableCustomizable() {
+ RegisterOptions("mutable", &mutable_, &mutable_option_info);
+ RegisterOptions("immutable", &immutable_, &immutable_option_info);
+ }
+ const char* Name() const override { return "MutableCustomizable"; }
+ };
+ MutableCustomizable mc, mc2;
+ std::string mismatch;
+ std::string opt_str;
+
+ ConfigOptions options = config_options_;
+ ASSERT_OK(mc.ConfigureOption(options, "mutable", "{id=B;}"));
+ options.mutable_options_only = true;
+ ASSERT_OK(mc.GetOptionString(options, &opt_str));
+ ASSERT_OK(mc2.ConfigureFromString(options, opt_str));
+ ASSERT_TRUE(mc.AreEquivalent(options, &mc2, &mismatch));
+
+ options.mutable_options_only = false;
+ ASSERT_OK(mc.ConfigureOption(options, "immutable", "{id=A; int=10}"));
+ auto* mm = mc.GetOptions<std::shared_ptr<TestCustomizable>>("mutable");
+ auto* im = mc.GetOptions<std::shared_ptr<TestCustomizable>>("immutable");
+ ASSERT_NE(mm, nullptr);
+ ASSERT_NE(mm->get(), nullptr);
+ ASSERT_NE(im, nullptr);
+ ASSERT_NE(im->get(), nullptr);
+
+ // Now only deal with mutable options
+ options.mutable_options_only = true;
+
+ // Setting nested immutable customizable options fails
+ ASSERT_NOK(mc.ConfigureOption(options, "immutable", "{id=B;}"));
+ ASSERT_NOK(mc.ConfigureOption(options, "immutable.id", "B"));
+ ASSERT_NOK(mc.ConfigureOption(options, "immutable.bool", "true"));
+ ASSERT_NOK(mc.ConfigureOption(options, "immutable", "bool=true"));
+ ASSERT_NOK(mc.ConfigureOption(options, "immutable", "{int=11;bool=true}"));
+ auto* im_a = im->get()->GetOptions<AOptions>("A");
+ ASSERT_NE(im_a, nullptr);
+ ASSERT_EQ(im_a->i, 10);
+ ASSERT_EQ(im_a->b, false);
+
+ // Setting nested mutable customizable options succeeds but the object did not
+ // change
+ ASSERT_OK(mc.ConfigureOption(options, "immutable.int", "11"));
+ ASSERT_EQ(im_a->i, 11);
+ ASSERT_EQ(im_a, im->get()->GetOptions<AOptions>("A"));
+
+ // The mutable configurable itself can be changed
+ ASSERT_OK(mc.ConfigureOption(options, "mutable.id", "A"));
+ ASSERT_OK(mc.ConfigureOption(options, "mutable", "A"));
+ ASSERT_OK(mc.ConfigureOption(options, "mutable", "{id=A}"));
+ ASSERT_OK(mc.ConfigureOption(options, "mutable", "{bool=true}"));
+
+ // The Nested options in the mutable object can be changed
+ ASSERT_OK(mc.ConfigureOption(options, "mutable", "{bool=true}"));
+ auto* mm_a = mm->get()->GetOptions<AOptions>("A");
+ ASSERT_EQ(mm_a->b, true);
+ ASSERT_OK(mc.ConfigureOption(options, "mutable", "{int=22;bool=false}"));
+ mm_a = mm->get()->GetOptions<AOptions>("A");
+ ASSERT_EQ(mm_a->i, 22);
+ ASSERT_EQ(mm_a->b, false);
+
+ // Only the mutable options should get serialized
+ options.mutable_options_only = false;
+ ASSERT_OK(mc.GetOptionString(options, &opt_str));
+ ASSERT_OK(mc.ConfigureOption(options, "immutable", "{id=B;}"));
+ options.mutable_options_only = true;
+
+ ASSERT_OK(mc.GetOptionString(options, &opt_str));
+ ASSERT_OK(mc2.ConfigureFromString(options, opt_str));
+ ASSERT_TRUE(mc.AreEquivalent(options, &mc2, &mismatch));
+ options.mutable_options_only = false;
+ ASSERT_FALSE(mc.AreEquivalent(options, &mc2, &mismatch));
+ ASSERT_EQ(mismatch, "immutable");
+}
+
+TEST_F(CustomizableTest, CustomManagedObjects) {
+ std::shared_ptr<TestCustomizable> object1, object2;
+ ASSERT_OK(LoadManagedObject<TestCustomizable>(
+ config_options_, "id=A_1;int=1;bool=true", &object1));
+ ASSERT_NE(object1, nullptr);
+ ASSERT_OK(
+ LoadManagedObject<TestCustomizable>(config_options_, "A_1", &object2));
+ ASSERT_EQ(object1, object2);
+ auto* opts = object2->GetOptions<AOptions>("A");
+ ASSERT_NE(opts, nullptr);
+ ASSERT_EQ(opts->i, 1);
+ ASSERT_EQ(opts->b, true);
+ ASSERT_OK(
+ LoadManagedObject<TestCustomizable>(config_options_, "A_2", &object2));
+ ASSERT_NE(object1, object2);
+ object1.reset();
+ ASSERT_OK(LoadManagedObject<TestCustomizable>(
+ config_options_, "id=A_1;int=2;bool=false", &object1));
+ opts = object1->GetOptions<AOptions>("A");
+ ASSERT_NE(opts, nullptr);
+ ASSERT_EQ(opts->i, 2);
+ ASSERT_EQ(opts->b, false);
+}
+
+TEST_F(CustomizableTest, CreateManagedObjects) {
+ class ManagedCustomizable : public Customizable {
+ public:
+ static const char* Type() { return "ManagedCustomizable"; }
+ static const char* kClassName() { return "Managed"; }
+ const char* Name() const override { return kClassName(); }
+ std::string GetId() const override { return id_; }
+ ManagedCustomizable() { id_ = GenerateIndividualId(); }
+ static Status CreateFromString(
+ const ConfigOptions& opts, const std::string& value,
+ std::shared_ptr<ManagedCustomizable>* result) {
+ return LoadManagedObject<ManagedCustomizable>(opts, value, result);
+ }
+
+ private:
+ std::string id_;
+ };
+
+ config_options_.registry->AddLibrary("Managed")
+ ->AddFactory<ManagedCustomizable>(
+ ObjectLibrary::PatternEntry::AsIndividualId(
+ ManagedCustomizable::kClassName()),
+ [](const std::string& /*name*/,
+ std::unique_ptr<ManagedCustomizable>* guard,
+ std::string* /* msg */) {
+ guard->reset(new ManagedCustomizable());
+ return guard->get();
+ });
+
+ std::shared_ptr<ManagedCustomizable> mc1, mc2, mc3, obj;
+ // Create a "deadbeef" customizable
+ std::string deadbeef =
+ std::string(ManagedCustomizable::kClassName()) + "@0xdeadbeef#0001";
+ ASSERT_OK(
+ ManagedCustomizable::CreateFromString(config_options_, deadbeef, &mc1));
+ // Create an object with the base/class name
+ ASSERT_OK(ManagedCustomizable::CreateFromString(
+ config_options_, ManagedCustomizable::kClassName(), &mc2));
+ // Creating another with the base name returns a different object
+ ASSERT_OK(ManagedCustomizable::CreateFromString(
+ config_options_, ManagedCustomizable::kClassName(), &mc3));
+ // At this point, there should be 4 managed objects (deadbeef, mc1, 2, and 3)
+ std::vector<std::shared_ptr<ManagedCustomizable>> objects;
+ ASSERT_OK(config_options_.registry->ListManagedObjects(&objects));
+ ASSERT_EQ(objects.size(), 4U);
+ objects.clear();
+ // Three separate object, none of them equal
+ ASSERT_NE(mc1, mc2);
+ ASSERT_NE(mc1, mc3);
+ ASSERT_NE(mc2, mc3);
+
+ // Creating another object with "deadbeef" object
+ ASSERT_OK(
+ ManagedCustomizable::CreateFromString(config_options_, deadbeef, &obj));
+ ASSERT_EQ(mc1, obj);
+ // Create another with the IDs of the instances
+ ASSERT_OK(ManagedCustomizable::CreateFromString(config_options_, mc1->GetId(),
+ &obj));
+ ASSERT_EQ(mc1, obj);
+ ASSERT_OK(ManagedCustomizable::CreateFromString(config_options_, mc2->GetId(),
+ &obj));
+ ASSERT_EQ(mc2, obj);
+ ASSERT_OK(ManagedCustomizable::CreateFromString(config_options_, mc3->GetId(),
+ &obj));
+ ASSERT_EQ(mc3, obj);
+
+ // Now get rid of deadbeef. 2 Objects left (m2+m3)
+ mc1.reset();
+ ASSERT_EQ(
+ config_options_.registry->GetManagedObject<ManagedCustomizable>(deadbeef),
+ nullptr);
+ ASSERT_OK(config_options_.registry->ListManagedObjects(&objects));
+ ASSERT_EQ(objects.size(), 2U);
+ objects.clear();
+
+ // Associate deadbeef with #2
+ ASSERT_OK(config_options_.registry->SetManagedObject(deadbeef, mc2));
+ ASSERT_OK(
+ ManagedCustomizable::CreateFromString(config_options_, deadbeef, &obj));
+ ASSERT_EQ(mc2, obj);
+ obj.reset();
+
+ // Get the ID of mc2 and then reset it. 1 Object left
+ std::string mc2id = mc2->GetId();
+ mc2.reset();
+ ASSERT_EQ(
+ config_options_.registry->GetManagedObject<ManagedCustomizable>(mc2id),
+ nullptr);
+ ASSERT_OK(config_options_.registry->ListManagedObjects(&objects));
+ ASSERT_EQ(objects.size(), 1U);
+ objects.clear();
+
+ // Create another object with the old mc2id.
+ ASSERT_OK(
+ ManagedCustomizable::CreateFromString(config_options_, mc2id, &mc2));
+ ASSERT_OK(
+ ManagedCustomizable::CreateFromString(config_options_, mc2id, &obj));
+ ASSERT_EQ(mc2, obj);
+
+ // For good measure, create another deadbeef object
+ ASSERT_OK(
+ ManagedCustomizable::CreateFromString(config_options_, deadbeef, &mc1));
+ ASSERT_OK(
+ ManagedCustomizable::CreateFromString(config_options_, deadbeef, &obj));
+ ASSERT_EQ(mc1, obj);
+}
+
+#endif // !ROCKSDB_LITE
+
+namespace {
+class TestSecondaryCache : public SecondaryCache {
+ public:
+ static const char* kClassName() { return "Test"; }
+ const char* Name() const override { return kClassName(); }
+ Status Insert(const Slice& /*key*/, void* /*value*/,
+ const Cache::CacheItemHelper* /*helper*/) override {
+ return Status::NotSupported();
+ }
+ std::unique_ptr<SecondaryCacheResultHandle> Lookup(
+ const Slice& /*key*/, const Cache::CreateCallback& /*create_cb*/,
+ bool /*wait*/, bool /*advise_erase*/, bool& is_in_sec_cache) override {
+ is_in_sec_cache = true;
+ return nullptr;
+ }
+
+ bool SupportForceErase() const override { return false; }
+
+ void Erase(const Slice& /*key*/) override {}
+
+ // Wait for a collection of handles to become ready
+ void WaitAll(std::vector<SecondaryCacheResultHandle*> /*handles*/) override {}
+
+ std::string GetPrintableOptions() const override { return ""; }
+};
+
+class TestStatistics : public StatisticsImpl {
+ public:
+ TestStatistics() : StatisticsImpl(nullptr) {}
+ const char* Name() const override { return kClassName(); }
+ static const char* kClassName() { return "Test"; }
+};
+
+class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
+ public:
+ TestFlushBlockPolicyFactory() {}
+
+ static const char* kClassName() { return "TestFlushBlockPolicyFactory"; }
+ const char* Name() const override { return kClassName(); }
+
+ FlushBlockPolicy* NewFlushBlockPolicy(
+ const BlockBasedTableOptions& /*table_options*/,
+ const BlockBuilder& /*data_block_builder*/) const override {
+ return nullptr;
+ }
+};
+
+class MockSliceTransform : public SliceTransform {
+ public:
+ const char* Name() const override { return kClassName(); }
+ static const char* kClassName() { return "Mock"; }
+
+ Slice Transform(const Slice& /*key*/) const override { return Slice(); }
+
+ bool InDomain(const Slice& /*key*/) const override { return false; }
+
+ bool InRange(const Slice& /*key*/) const override { return false; }
+};
+
+class MockMemoryAllocator : public BaseMemoryAllocator {
+ public:
+ static const char* kClassName() { return "MockMemoryAllocator"; }
+ const char* Name() const override { return kClassName(); }
+};
+
+#ifndef ROCKSDB_LITE
+class MockEncryptionProvider : public EncryptionProvider {
+ public:
+ explicit MockEncryptionProvider(const std::string& id) : id_(id) {}
+ static const char* kClassName() { return "Mock"; }
+ const char* Name() const override { return kClassName(); }
+ size_t GetPrefixLength() const override { return 0; }
+ Status CreateNewPrefix(const std::string& /*fname*/, char* /*prefix*/,
+ size_t /*prefixLength*/) const override {
+ return Status::NotSupported();
+ }
+
+ Status AddCipher(const std::string& /*descriptor*/, const char* /*cipher*/,
+ size_t /*len*/, bool /*for_write*/) override {
+ return Status::NotSupported();
+ }
+
+ Status CreateCipherStream(
+ const std::string& /*fname*/, const EnvOptions& /*options*/,
+ Slice& /*prefix*/,
+ std::unique_ptr<BlockAccessCipherStream>* /*result*/) override {
+ return Status::NotSupported();
+ }
+ Status ValidateOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts) const override {
+ if (EndsWith(id_, "://test")) {
+ return EncryptionProvider::ValidateOptions(db_opts, cf_opts);
+ } else {
+ return Status::InvalidArgument("MockProvider not initialized");
+ }
+ }
+
+ private:
+ std::string id_;
+};
+
+class MockCipher : public BlockCipher {
+ public:
+ const char* Name() const override { return "Mock"; }
+ size_t BlockSize() override { return 0; }
+ Status Encrypt(char* /*data*/) override { return Status::NotSupported(); }
+ Status Decrypt(char* data) override { return Encrypt(data); }
+};
+#endif // ROCKSDB_LITE
+
+class DummyFileSystem : public FileSystemWrapper {
+ public:
+ explicit DummyFileSystem(const std::shared_ptr<FileSystem>& t)
+ : FileSystemWrapper(t) {}
+ static const char* kClassName() { return "DummyFileSystem"; }
+ const char* Name() const override { return kClassName(); }
+};
+
+#ifndef ROCKSDB_LITE
+
+#endif // ROCKSDB_LITE
+
+class MockTablePropertiesCollectorFactory
+ : public TablePropertiesCollectorFactory {
+ private:
+ public:
+ TablePropertiesCollector* CreateTablePropertiesCollector(
+ TablePropertiesCollectorFactory::Context /*context*/) override {
+ return nullptr;
+ }
+ static const char* kClassName() { return "Mock"; }
+ const char* Name() const override { return kClassName(); }
+};
+
+class MockSstPartitionerFactory : public SstPartitionerFactory {
+ public:
+ static const char* kClassName() { return "Mock"; }
+ const char* Name() const override { return kClassName(); }
+ std::unique_ptr<SstPartitioner> CreatePartitioner(
+ const SstPartitioner::Context& /* context */) const override {
+ return nullptr;
+ }
+};
+
+class MockFileChecksumGenFactory : public FileChecksumGenFactory {
+ public:
+ static const char* kClassName() { return "Mock"; }
+ const char* Name() const override { return kClassName(); }
+ std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
+ const FileChecksumGenContext& /*context*/) override {
+ return nullptr;
+ }
+};
+
+class MockFilterPolicy : public FilterPolicy {
+ public:
+ static const char* kClassName() { return "MockFilterPolicy"; }
+ const char* Name() const override { return kClassName(); }
+ const char* CompatibilityName() const override { return Name(); }
+ FilterBitsBuilder* GetBuilderWithContext(
+ const FilterBuildingContext&) const override {
+ return nullptr;
+ }
+ FilterBitsReader* GetFilterBitsReader(
+ const Slice& /*contents*/) const override {
+ return nullptr;
+ }
+};
+
+#ifndef ROCKSDB_LITE
+static int RegisterLocalObjects(ObjectLibrary& library,
+ const std::string& /*arg*/) {
+ size_t num_types;
+ library.AddFactory<TableFactory>(
+ mock::MockTableFactory::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<TableFactory>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new mock::MockTableFactory());
+ return guard->get();
+ });
+ library.AddFactory<EventListener>(
+ OnFileDeletionListener::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<EventListener>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new OnFileDeletionListener());
+ return guard->get();
+ });
+ library.AddFactory<EventListener>(
+ FlushCounterListener::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<EventListener>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new FlushCounterListener());
+ return guard->get();
+ });
+ // Load any locally defined objects here
+ library.AddFactory<const SliceTransform>(
+ MockSliceTransform::kClassName(),
+ [](const std::string& /*uri*/,
+ std::unique_ptr<const SliceTransform>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockSliceTransform());
+ return guard->get();
+ });
+ library.AddFactory<Statistics>(
+ TestStatistics::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<Statistics>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new TestStatistics());
+ return guard->get();
+ });
+
+ library.AddFactory<EncryptionProvider>(
+ ObjectLibrary::PatternEntry(MockEncryptionProvider::kClassName(), true)
+ .AddSuffix("://test"),
+ [](const std::string& uri, std::unique_ptr<EncryptionProvider>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockEncryptionProvider(uri));
+ return guard->get();
+ });
+ library.AddFactory<BlockCipher>(
+ "Mock",
+ [](const std::string& /*uri*/, std::unique_ptr<BlockCipher>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockCipher());
+ return guard->get();
+ });
+ library.AddFactory<MemoryAllocator>(
+ MockMemoryAllocator::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<MemoryAllocator>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockMemoryAllocator());
+ return guard->get();
+ });
+ library.AddFactory<FlushBlockPolicyFactory>(
+ TestFlushBlockPolicyFactory::kClassName(),
+ [](const std::string& /*uri*/,
+ std::unique_ptr<FlushBlockPolicyFactory>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new TestFlushBlockPolicyFactory());
+ return guard->get();
+ });
+
+ library.AddFactory<SecondaryCache>(
+ TestSecondaryCache::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<SecondaryCache>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new TestSecondaryCache());
+ return guard->get();
+ });
+
+ library.AddFactory<FileSystem>(
+ DummyFileSystem::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new DummyFileSystem(nullptr));
+ return guard->get();
+ });
+
+ library.AddFactory<SstPartitionerFactory>(
+ MockSstPartitionerFactory::kClassName(),
+ [](const std::string& /*uri*/,
+ std::unique_ptr<SstPartitionerFactory>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockSstPartitionerFactory());
+ return guard->get();
+ });
+
+ library.AddFactory<FileChecksumGenFactory>(
+ MockFileChecksumGenFactory::kClassName(),
+ [](const std::string& /*uri*/,
+ std::unique_ptr<FileChecksumGenFactory>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockFileChecksumGenFactory());
+ return guard->get();
+ });
+
+ library.AddFactory<TablePropertiesCollectorFactory>(
+ MockTablePropertiesCollectorFactory::kClassName(),
+ [](const std::string& /*uri*/,
+ std::unique_ptr<TablePropertiesCollectorFactory>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockTablePropertiesCollectorFactory());
+ return guard->get();
+ });
+
+ library.AddFactory<const FilterPolicy>(
+ MockFilterPolicy::kClassName(),
+ [](const std::string& /*uri*/, std::unique_ptr<const FilterPolicy>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new MockFilterPolicy());
+ return guard->get();
+ });
+
+ return static_cast<int>(library.GetFactoryCount(&num_types));
+}
+#endif // !ROCKSDB_LITE
+} // namespace
+
+class LoadCustomizableTest : public testing::Test {
+ public:
+ LoadCustomizableTest() {
+ config_options_.ignore_unsupported_options = false;
+ config_options_.invoke_prepare_options = false;
+ }
+ bool RegisterTests(const std::string& arg) {
+#ifndef ROCKSDB_LITE
+ config_options_.registry->AddLibrary("custom-tests",
+ test::RegisterTestObjects, arg);
+ config_options_.registry->AddLibrary("local-tests", RegisterLocalObjects,
+ arg);
+ return true;
+#else
+ (void)arg;
+ return false;
+#endif // !ROCKSDB_LITE
+ }
+
+ template <typename T, typename U>
+ Status TestCreateStatic(const std::string& name, U** result,
+ bool delete_result = false) {
+ Status s = T::CreateFromString(config_options_, name, result);
+ if (s.ok()) {
+ EXPECT_NE(*result, nullptr);
+ EXPECT_TRUE(*result != nullptr && (*result)->IsInstanceOf(name));
+ }
+ if (delete_result) {
+ delete *result;
+ *result = nullptr;
+ }
+ return s;
+ }
+
+ template <typename T, typename U>
+ std::shared_ptr<U> ExpectCreateShared(const std::string& name,
+ std::shared_ptr<U>* object) {
+ EXPECT_OK(T::CreateFromString(config_options_, name, object));
+ EXPECT_NE(object->get(), nullptr);
+ EXPECT_TRUE(object->get()->IsInstanceOf(name));
+ return *object;
+ }
+
+ template <typename T>
+ std::shared_ptr<T> ExpectCreateShared(const std::string& name) {
+ std::shared_ptr<T> result;
+ return ExpectCreateShared<T>(name, &result);
+ }
+
+ template <typename T, typename U>
+ Status TestExpectedBuiltins(
+ const std::string& mock, const std::unordered_set<std::string>& expected,
+ std::shared_ptr<U>* object, std::vector<std::string>* failed,
+ const std::function<std::vector<std::string>(const std::string&)>& alt =
+ nullptr) {
+ std::unordered_set<std::string> factories = expected;
+ Status s = T::CreateFromString(config_options_, mock, object);
+ EXPECT_NOK(s);
+#ifndef ROCKSDB_LITE
+ std::vector<std::string> builtins;
+ ObjectLibrary::Default()->GetFactoryNames(T::Type(), &builtins);
+ factories.insert(builtins.begin(), builtins.end());
+#endif // ROCKSDB_LITE
+ Status result;
+ int created = 0;
+ for (const auto& name : factories) {
+ created++;
+ s = T::CreateFromString(config_options_, name, object);
+ if (!s.ok() && alt != nullptr) {
+ for (const auto& alt_name : alt(name)) {
+ s = T::CreateFromString(config_options_, alt_name, object);
+ if (s.ok()) {
+ break;
+ }
+ }
+ }
+ if (!s.ok()) {
+ result = s;
+ failed->push_back(name);
+ } else {
+ EXPECT_NE(object->get(), nullptr);
+ EXPECT_TRUE(object->get()->IsInstanceOf(name));
+ }
+ }
+#ifndef ROCKSDB_LITE
+ std::vector<std::string> plugins;
+ ObjectRegistry::Default()->GetFactoryNames(T::Type(), &plugins);
+ if (plugins.size() > builtins.size()) {
+ for (const auto& name : plugins) {
+ if (factories.find(name) == factories.end()) {
+ created++;
+ s = T::CreateFromString(config_options_, name, object);
+ if (!s.ok() && alt != nullptr) {
+ for (const auto& alt_name : alt(name)) {
+ s = T::CreateFromString(config_options_, alt_name, object);
+ if (s.ok()) {
+ break;
+ }
+ }
+ }
+ if (!s.ok()) {
+ failed->push_back(name);
+ if (result.ok()) {
+ result = s;
+ }
+ printf("%s: Failed creating plugin[%s]: %s\n", T::Type(),
+ name.c_str(), s.ToString().c_str());
+ } else if (object->get() == nullptr ||
+ !object->get()->IsInstanceOf(name)) {
+ failed->push_back(name);
+ printf("%s: Invalid plugin[%s]\n", T::Type(), name.c_str());
+ }
+ }
+ }
+ }
+ printf("%s: Created %d (expected+builtins+plugins %d+%d+%d) %d Failed\n",
+ T::Type(), created, (int)expected.size(),
+ (int)(factories.size() - expected.size()),
+ (int)(plugins.size() - builtins.size()), (int)failed->size());
+#else
+ printf("%s: Created %d (expected %d) %d Failed\n", T::Type(), created,
+ (int)expected.size(), (int)failed->size());
+#endif // ROCKSDB_LITE
+ return result;
+ }
+
+ template <typename T>
+ Status TestSharedBuiltins(const std::string& mock,
+ const std::string& expected,
+ std::vector<std::string>* failed = nullptr) {
+ std::unordered_set<std::string> values;
+ if (!expected.empty()) {
+ values.insert(expected);
+ }
+ std::shared_ptr<T> object;
+ if (failed != nullptr) {
+ return TestExpectedBuiltins<T>(mock, values, &object, failed);
+ } else {
+ std::vector<std::string> failures;
+ Status s = TestExpectedBuiltins<T>(mock, values, &object, &failures);
+ EXPECT_EQ(0U, failures.size());
+ return s;
+ }
+ }
+
+ template <typename T, typename U>
+ Status TestStaticBuiltins(const std::string& mock, U** object,
+ const std::unordered_set<std::string>& expected,
+ std::vector<std::string>* failed,
+ bool delete_objects = false) {
+ std::unordered_set<std::string> factories = expected;
+ Status s = TestCreateStatic<T>(mock, object, delete_objects);
+ EXPECT_NOK(s);
+#ifndef ROCKSDB_LITE
+ std::vector<std::string> builtins;
+ ObjectLibrary::Default()->GetFactoryNames(T::Type(), &builtins);
+ factories.insert(builtins.begin(), builtins.end());
+#endif // ROCKSDB_LITE
+ int created = 0;
+ Status result;
+ for (const auto& name : factories) {
+ created++;
+ s = TestCreateStatic<T>(name, object, delete_objects);
+ if (!s.ok()) {
+ result = s;
+ failed->push_back(name);
+ }
+ }
+#ifndef ROCKSDB_LITE
+ std::vector<std::string> plugins;
+ ObjectRegistry::Default()->GetFactoryNames(T::Type(), &plugins);
+ if (plugins.size() > builtins.size()) {
+ for (const auto& name : plugins) {
+ if (factories.find(name) == factories.end()) {
+ created++;
+ s = T::CreateFromString(config_options_, name, object);
+ if (!s.ok() || *object == nullptr ||
+ !((*object)->IsInstanceOf(name))) {
+ failed->push_back(name);
+ if (result.ok() && !s.ok()) {
+ result = s;
+ }
+ printf("%s: Failed creating plugin[%s]: %s\n", T::Type(),
+ name.c_str(), s.ToString().c_str());
+ }
+ if (delete_objects) {
+ delete *object;
+ *object = nullptr;
+ }
+ }
+ }
+ }
+ printf("%s: Created %d (expected+builtins+plugins %d+%d+%d) %d Failed\n",
+ T::Type(), created, (int)expected.size(),
+ (int)(factories.size() - expected.size()),
+ (int)(plugins.size() - builtins.size()), (int)failed->size());
+#else
+ printf("%s: Created %d (expected %d) %d Failed\n", T::Type(), created,
+ (int)expected.size(), (int)failed->size());
+#endif // ROCKSDB_LITE
+ return result;
+ }
+
+ protected:
+ DBOptions db_opts_;
+ ColumnFamilyOptions cf_opts_;
+ ConfigOptions config_options_;
+};
+
+TEST_F(LoadCustomizableTest, LoadTableFactoryTest) {
+ ASSERT_OK(
+ TestSharedBuiltins<TableFactory>(mock::MockTableFactory::kClassName(),
+ TableFactory::kBlockBasedTableName()));
+#ifndef ROCKSDB_LITE
+ std::string opts_str = "table_factory=";
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options_, cf_opts_,
+ opts_str + TableFactory::kBlockBasedTableName(), &cf_opts_));
+ ASSERT_NE(cf_opts_.table_factory.get(), nullptr);
+ ASSERT_STREQ(cf_opts_.table_factory->Name(),
+ TableFactory::kBlockBasedTableName());
+#endif // ROCKSDB_LITE
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<TableFactory>(mock::MockTableFactory::kClassName());
+#ifndef ROCKSDB_LITE
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options_, cf_opts_,
+ opts_str + mock::MockTableFactory::kClassName(), &cf_opts_));
+ ASSERT_NE(cf_opts_.table_factory.get(), nullptr);
+ ASSERT_STREQ(cf_opts_.table_factory->Name(),
+ mock::MockTableFactory::kClassName());
+#endif // ROCKSDB_LITE
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadFileSystemTest) {
+ ASSERT_OK(TestSharedBuiltins<FileSystem>(DummyFileSystem::kClassName(),
+ FileSystem::kDefaultName()));
+ if (RegisterTests("Test")) {
+ auto fs = ExpectCreateShared<FileSystem>(DummyFileSystem::kClassName());
+ ASSERT_FALSE(fs->IsInstanceOf(FileSystem::kDefaultName()));
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) {
+ ASSERT_OK(
+ TestSharedBuiltins<SecondaryCache>(TestSecondaryCache::kClassName(), ""));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<SecondaryCache>(TestSecondaryCache::kClassName());
+ }
+}
+
+#ifndef ROCKSDB_LITE
+TEST_F(LoadCustomizableTest, LoadSstPartitionerFactoryTest) {
+ ASSERT_OK(TestSharedBuiltins<SstPartitionerFactory>(
+ "Mock", SstPartitionerFixedPrefixFactory::kClassName()));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<SstPartitionerFactory>("Mock");
+ }
+}
+#endif // ROCKSDB_LITE
+
+TEST_F(LoadCustomizableTest, LoadChecksumGenFactoryTest) {
+ ASSERT_OK(TestSharedBuiltins<FileChecksumGenFactory>("Mock", ""));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<FileChecksumGenFactory>("Mock");
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadTablePropertiesCollectorFactoryTest) {
+ ASSERT_OK(TestSharedBuiltins<TablePropertiesCollectorFactory>(
+ MockTablePropertiesCollectorFactory::kClassName(), ""));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<TablePropertiesCollectorFactory>(
+ MockTablePropertiesCollectorFactory::kClassName());
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadComparatorTest) {
+ const Comparator* bytewise = BytewiseComparator();
+ const Comparator* reverse = ReverseBytewiseComparator();
+ const Comparator* result = nullptr;
+ std::unordered_set<std::string> expected = {bytewise->Name(),
+ reverse->Name()};
+ std::vector<std::string> failures;
+ ASSERT_OK(TestStaticBuiltins<Comparator>(
+ test::SimpleSuffixReverseComparator::kClassName(), &result, expected,
+ &failures));
+ if (RegisterTests("Test")) {
+ ASSERT_OK(TestCreateStatic<Comparator>(
+ test::SimpleSuffixReverseComparator::kClassName(), &result));
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadSliceTransformFactoryTest) {
+ std::shared_ptr<const SliceTransform> result;
+ std::vector<std::string> failures;
+ std::unordered_set<std::string> expected = {"rocksdb.Noop", "fixed",
+ "rocksdb.FixedPrefix", "capped",
+ "rocksdb.CappedPrefix"};
+ ASSERT_OK(TestExpectedBuiltins<SliceTransform>(
+ "Mock", expected, &result, &failures, [](const std::string& name) {
+ std::vector<std::string> names = {name + ":22", name + ".22"};
+ return names;
+ }));
+ ASSERT_OK(SliceTransform::CreateFromString(
+ config_options_, "rocksdb.FixedPrefix.22", &result));
+ ASSERT_NE(result.get(), nullptr);
+ ASSERT_TRUE(result->IsInstanceOf("fixed"));
+ ASSERT_OK(SliceTransform::CreateFromString(
+ config_options_, "rocksdb.CappedPrefix.22", &result));
+ ASSERT_NE(result.get(), nullptr);
+ ASSERT_TRUE(result->IsInstanceOf("capped"));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<SliceTransform>("Mock", &result);
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadStatisticsTest) {
+ ASSERT_OK(TestSharedBuiltins<Statistics>(TestStatistics::kClassName(),
+ "BasicStatistics"));
+ // Empty will create a default BasicStatistics
+ ASSERT_OK(
+ Statistics::CreateFromString(config_options_, "", &db_opts_.statistics));
+ ASSERT_NE(db_opts_.statistics, nullptr);
+ ASSERT_STREQ(db_opts_.statistics->Name(), "BasicStatistics");
+
+#ifndef ROCKSDB_LITE
+ ASSERT_NOK(GetDBOptionsFromString(config_options_, db_opts_,
+ "statistics=Test", &db_opts_));
+ ASSERT_OK(GetDBOptionsFromString(config_options_, db_opts_,
+ "statistics=BasicStatistics", &db_opts_));
+ ASSERT_NE(db_opts_.statistics, nullptr);
+ ASSERT_STREQ(db_opts_.statistics->Name(), "BasicStatistics");
+
+ if (RegisterTests("test")) {
+ auto stats = ExpectCreateShared<Statistics>(TestStatistics::kClassName());
+
+ ASSERT_OK(GetDBOptionsFromString(config_options_, db_opts_,
+ "statistics=Test", &db_opts_));
+ ASSERT_NE(db_opts_.statistics, nullptr);
+ ASSERT_STREQ(db_opts_.statistics->Name(), TestStatistics::kClassName());
+
+ ASSERT_OK(GetDBOptionsFromString(
+ config_options_, db_opts_, "statistics={id=Test;inner=BasicStatistics}",
+ &db_opts_));
+ ASSERT_NE(db_opts_.statistics, nullptr);
+ ASSERT_STREQ(db_opts_.statistics->Name(), TestStatistics::kClassName());
+ auto* inner = db_opts_.statistics->GetOptions<std::shared_ptr<Statistics>>(
+ "StatisticsOptions");
+ ASSERT_NE(inner, nullptr);
+ ASSERT_NE(inner->get(), nullptr);
+ ASSERT_STREQ(inner->get()->Name(), "BasicStatistics");
+
+ ASSERT_OK(Statistics::CreateFromString(
+ config_options_, "id=BasicStatistics;inner=Test", &stats));
+ ASSERT_NE(stats, nullptr);
+ ASSERT_STREQ(stats->Name(), "BasicStatistics");
+ inner = stats->GetOptions<std::shared_ptr<Statistics>>("StatisticsOptions");
+ ASSERT_NE(inner, nullptr);
+ ASSERT_NE(inner->get(), nullptr);
+ ASSERT_STREQ(inner->get()->Name(), TestStatistics::kClassName());
+ }
+#endif
+}
+
+TEST_F(LoadCustomizableTest, LoadMemTableRepFactoryTest) {
+ std::unordered_set<std::string> expected = {
+ SkipListFactory::kClassName(),
+ SkipListFactory::kNickName(),
+ };
+
+ std::vector<std::string> failures;
+ std::shared_ptr<MemTableRepFactory> factory;
+ Status s = TestExpectedBuiltins<MemTableRepFactory>(
+ "SpecialSkipListFactory", expected, &factory, &failures);
+ // There is a "cuckoo" factory registered that we expect to fail. Ignore the
+ // error if this is the one
+ if (s.ok() || failures.size() > 1 || failures[0] != "cuckoo") {
+ ASSERT_OK(s);
+ }
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<MemTableRepFactory>("SpecialSkipListFactory");
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadMergeOperatorTest) {
+ std::shared_ptr<MergeOperator> result;
+ std::vector<std::string> failed;
+ std::unordered_set<std::string> expected = {
+ "put", "put_v1", "PutOperator", "uint64add", "UInt64AddOperator",
+ "max", "MaxOperator",
+ };
+#ifndef ROCKSDB_LITE
+ expected.insert({
+ StringAppendOperator::kClassName(),
+ StringAppendOperator::kNickName(),
+ StringAppendTESTOperator::kClassName(),
+ StringAppendTESTOperator::kNickName(),
+ SortList::kClassName(),
+ SortList::kNickName(),
+ BytesXOROperator::kClassName(),
+ BytesXOROperator::kNickName(),
+ });
+#endif // ROCKSDB_LITE
+
+ ASSERT_OK(TestExpectedBuiltins<MergeOperator>("Changling", expected, &result,
+ &failed));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<MergeOperator>("Changling");
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadCompactionFilterFactoryTest) {
+ ASSERT_OK(TestSharedBuiltins<CompactionFilterFactory>("Changling", ""));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<CompactionFilterFactory>("Changling");
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadCompactionFilterTest) {
+ const CompactionFilter* result = nullptr;
+ std::vector<std::string> failures;
+ ASSERT_OK(TestStaticBuiltins<CompactionFilter>("Changling", &result, {},
+ &failures, true));
+ if (RegisterTests("Test")) {
+ ASSERT_OK(TestCreateStatic<CompactionFilter>("Changling", &result, true));
+ }
+}
+
+#ifndef ROCKSDB_LITE
+TEST_F(LoadCustomizableTest, LoadEventListenerTest) {
+ ASSERT_OK(TestSharedBuiltins<EventListener>(
+ OnFileDeletionListener::kClassName(), ""));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<EventListener>(OnFileDeletionListener::kClassName());
+ ExpectCreateShared<EventListener>(FlushCounterListener::kClassName());
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadEncryptionProviderTest) {
+ std::vector<std::string> failures;
+ std::shared_ptr<EncryptionProvider> result;
+ ASSERT_OK(
+ TestExpectedBuiltins<EncryptionProvider>("Mock", {}, &result, &failures));
+ if (!failures.empty()) {
+ ASSERT_EQ(failures[0], "1://test");
+ ASSERT_EQ(failures.size(), 1U);
+ }
+
+ result = ExpectCreateShared<EncryptionProvider>("CTR");
+ ASSERT_NOK(result->ValidateOptions(db_opts_, cf_opts_));
+ ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, "CTR://test",
+ &result));
+ ASSERT_NE(result, nullptr);
+ ASSERT_STREQ(result->Name(), "CTR");
+ ASSERT_OK(result->ValidateOptions(db_opts_, cf_opts_));
+
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<EncryptionProvider>("Mock");
+ ASSERT_OK(EncryptionProvider::CreateFromString(config_options_,
+ "Mock://test", &result));
+ ASSERT_NE(result, nullptr);
+ ASSERT_STREQ(result->Name(), "Mock");
+ ASSERT_OK(result->ValidateOptions(db_opts_, cf_opts_));
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadEncryptionCipherTest) {
+ ASSERT_OK(TestSharedBuiltins<BlockCipher>("Mock", "ROT13"));
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<BlockCipher>("Mock");
+ }
+}
+#endif // !ROCKSDB_LITE
+
+TEST_F(LoadCustomizableTest, LoadSystemClockTest) {
+ ASSERT_OK(TestSharedBuiltins<SystemClock>(MockSystemClock::kClassName(),
+ SystemClock::kDefaultName()));
+ if (RegisterTests("Test")) {
+ auto result =
+ ExpectCreateShared<SystemClock>(MockSystemClock::kClassName());
+ ASSERT_FALSE(result->IsInstanceOf(SystemClock::kDefaultName()));
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadMemoryAllocatorTest) {
+ std::vector<std::string> failures;
+ Status s = TestSharedBuiltins<MemoryAllocator>(
+ MockMemoryAllocator::kClassName(), DefaultMemoryAllocator::kClassName(),
+ &failures);
+ if (failures.empty()) {
+ ASSERT_OK(s);
+ } else {
+ ASSERT_NOK(s);
+ for (const auto& failure : failures) {
+ if (failure == JemallocNodumpAllocator::kClassName()) {
+ ASSERT_FALSE(JemallocNodumpAllocator::IsSupported());
+ } else if (failure == MemkindKmemAllocator::kClassName()) {
+ ASSERT_FALSE(MemkindKmemAllocator::IsSupported());
+ } else {
+ printf("BYPASSED: %s -- %s\n", failure.c_str(), s.ToString().c_str());
+ }
+ }
+ }
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<MemoryAllocator>(MockMemoryAllocator::kClassName());
+ }
+}
+
+TEST_F(LoadCustomizableTest, LoadFilterPolicyTest) {
+ const std::string kAutoBloom = BloomFilterPolicy::kClassName();
+ const std::string kAutoRibbon = RibbonFilterPolicy::kClassName();
+
+ std::shared_ptr<const FilterPolicy> result;
+ std::vector<std::string> failures;
+ std::unordered_set<std::string> expected = {
+ ReadOnlyBuiltinFilterPolicy::kClassName(),
+ };
+
+#ifndef ROCKSDB_LITE
+ expected.insert({
+ kAutoBloom,
+ BloomFilterPolicy::kNickName(),
+ kAutoRibbon,
+ RibbonFilterPolicy::kNickName(),
+ });
+#endif // ROCKSDB_LITE
+ ASSERT_OK(TestExpectedBuiltins<const FilterPolicy>(
+ "Mock", expected, &result, &failures, [](const std::string& name) {
+ std::vector<std::string> names = {name + ":1.234"};
+ return names;
+ }));
+#ifndef ROCKSDB_LITE
+ ASSERT_OK(FilterPolicy::CreateFromString(
+ config_options_, kAutoBloom + ":1.234:false", &result));
+ ASSERT_NE(result.get(), nullptr);
+ ASSERT_TRUE(result->IsInstanceOf(kAutoBloom));
+ ASSERT_OK(FilterPolicy::CreateFromString(
+ config_options_, kAutoBloom + ":1.234:false", &result));
+ ASSERT_NE(result.get(), nullptr);
+ ASSERT_TRUE(result->IsInstanceOf(kAutoBloom));
+ ASSERT_OK(FilterPolicy::CreateFromString(config_options_,
+ kAutoRibbon + ":1.234:-1", &result));
+ ASSERT_NE(result.get(), nullptr);
+ ASSERT_TRUE(result->IsInstanceOf(kAutoRibbon));
+ ASSERT_OK(FilterPolicy::CreateFromString(config_options_,
+ kAutoRibbon + ":1.234:56", &result));
+ ASSERT_NE(result.get(), nullptr);
+ ASSERT_TRUE(result->IsInstanceOf(kAutoRibbon));
+#endif // ROCKSDB_LITE
+
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<FilterPolicy>(MockFilterPolicy::kClassName(), &result);
+ }
+
+ std::shared_ptr<TableFactory> table;
+
+#ifndef ROCKSDB_LITE
+ std::string table_opts = "id=BlockBasedTable; filter_policy=";
+ ASSERT_OK(TableFactory::CreateFromString(config_options_,
+ table_opts + "nullptr", &table));
+ ASSERT_NE(table.get(), nullptr);
+ auto bbto = table->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_EQ(bbto->filter_policy.get(), nullptr);
+ ASSERT_OK(TableFactory::CreateFromString(
+ config_options_, table_opts + ReadOnlyBuiltinFilterPolicy::kClassName(),
+ &table));
+ bbto = table->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_NE(bbto->filter_policy.get(), nullptr);
+ ASSERT_STREQ(bbto->filter_policy->Name(),
+ ReadOnlyBuiltinFilterPolicy::kClassName());
+ ASSERT_OK(TableFactory::CreateFromString(
+ config_options_, table_opts + MockFilterPolicy::kClassName(), &table));
+ bbto = table->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_NE(bbto->filter_policy.get(), nullptr);
+ ASSERT_TRUE(
+ bbto->filter_policy->IsInstanceOf(MockFilterPolicy::kClassName()));
+#endif // ROCKSDB_LITE
+}
+
+TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) {
+ std::shared_ptr<FlushBlockPolicyFactory> result;
+ std::shared_ptr<TableFactory> table;
+ std::vector<std::string> failed;
+ std::unordered_set<std::string> expected = {
+ FlushBlockBySizePolicyFactory::kClassName(),
+ FlushBlockEveryKeyPolicyFactory::kClassName(),
+ };
+
+ ASSERT_OK(TestExpectedBuiltins<FlushBlockPolicyFactory>(
+ TestFlushBlockPolicyFactory::kClassName(), expected, &result, &failed));
+
+ // An empty policy name creates a BySize policy
+ ASSERT_OK(
+ FlushBlockPolicyFactory::CreateFromString(config_options_, "", &result));
+ ASSERT_NE(result, nullptr);
+ ASSERT_STREQ(result->Name(), FlushBlockBySizePolicyFactory::kClassName());
+
+#ifndef ROCKSDB_LITE
+ std::string table_opts = "id=BlockBasedTable; flush_block_policy_factory=";
+ ASSERT_OK(TableFactory::CreateFromString(
+ config_options_,
+ table_opts + FlushBlockEveryKeyPolicyFactory::kClassName(), &table));
+ auto bbto = table->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_NE(bbto->flush_block_policy_factory.get(), nullptr);
+ ASSERT_STREQ(bbto->flush_block_policy_factory->Name(),
+ FlushBlockEveryKeyPolicyFactory::kClassName());
+ if (RegisterTests("Test")) {
+ ExpectCreateShared<FlushBlockPolicyFactory>(
+ TestFlushBlockPolicyFactory::kClassName());
+ ASSERT_OK(TableFactory::CreateFromString(
+ config_options_, table_opts + TestFlushBlockPolicyFactory::kClassName(),
+ &table));
+ bbto = table->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_NE(bbto->flush_block_policy_factory.get(), nullptr);
+ ASSERT_STREQ(bbto->flush_block_policy_factory->Name(),
+ TestFlushBlockPolicyFactory::kClassName());
+ }
+#endif // ROCKSDB_LITE
+}
+
+} // namespace ROCKSDB_NAMESPACE
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+#ifdef GFLAGS
+ ParseCommandLineFlags(&argc, &argv, true);
+#endif // GFLAGS
+ return RUN_ALL_TESTS();
+}
diff --git a/src/rocksdb/options/db_options.cc b/src/rocksdb/options/db_options.cc
new file mode 100644
index 000000000..e0bc892fc
--- /dev/null
+++ b/src/rocksdb/options/db_options.cc
@@ -0,0 +1,1086 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "options/db_options.h"
+
+#include <cinttypes>
+
+#include "logging/logging.h"
+#include "options/configurable_helper.h"
+#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "port/port.h"
+#include "rocksdb/configurable.h"
+#include "rocksdb/env.h"
+#include "rocksdb/file_system.h"
+#include "rocksdb/listener.h"
+#include "rocksdb/rate_limiter.h"
+#include "rocksdb/sst_file_manager.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/system_clock.h"
+#include "rocksdb/utilities/options_type.h"
+#include "rocksdb/wal_filter.h"
+#include "util/string_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+#ifndef ROCKSDB_LITE
+static std::unordered_map<std::string, WALRecoveryMode>
+ wal_recovery_mode_string_map = {
+ {"kTolerateCorruptedTailRecords",
+ WALRecoveryMode::kTolerateCorruptedTailRecords},
+ {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency},
+ {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery},
+ {"kSkipAnyCorruptedRecords",
+ WALRecoveryMode::kSkipAnyCorruptedRecords}};
+
+static std::unordered_map<std::string, DBOptions::AccessHint>
+ access_hint_string_map = {{"NONE", DBOptions::AccessHint::NONE},
+ {"NORMAL", DBOptions::AccessHint::NORMAL},
+ {"SEQUENTIAL", DBOptions::AccessHint::SEQUENTIAL},
+ {"WILLNEED", DBOptions::AccessHint::WILLNEED}};
+
+static std::unordered_map<std::string, CacheTier> cache_tier_string_map = {
+ {"kVolatileTier", CacheTier::kVolatileTier},
+ {"kNonVolatileBlockTier", CacheTier::kNonVolatileBlockTier}};
+
+static std::unordered_map<std::string, InfoLogLevel> info_log_level_string_map =
+ {{"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL},
+ {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL},
+ {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL},
+ {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL},
+ {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
+ {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};
+
+static std::unordered_map<std::string, OptionTypeInfo>
+ db_mutable_options_type_info = {
+ {"allow_os_buffer",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"base_background_compactions",
+ {0, OptionType::kInt, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kMutable}},
+ {"max_background_jobs",
+ {offsetof(struct MutableDBOptions, max_background_jobs),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_background_compactions",
+ {offsetof(struct MutableDBOptions, max_background_compactions),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_subcompactions",
+ {offsetof(struct MutableDBOptions, max_subcompactions),
+ OptionType::kUInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"avoid_flush_during_shutdown",
+ {offsetof(struct MutableDBOptions, avoid_flush_during_shutdown),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"writable_file_max_buffer_size",
+ {offsetof(struct MutableDBOptions, writable_file_max_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"delayed_write_rate",
+ {offsetof(struct MutableDBOptions, delayed_write_rate),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_total_wal_size",
+ {offsetof(struct MutableDBOptions, max_total_wal_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"delete_obsolete_files_period_micros",
+ {offsetof(struct MutableDBOptions,
+ delete_obsolete_files_period_micros),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"stats_dump_period_sec",
+ {offsetof(struct MutableDBOptions, stats_dump_period_sec),
+ OptionType::kUInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"stats_persist_period_sec",
+ {offsetof(struct MutableDBOptions, stats_persist_period_sec),
+ OptionType::kUInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"stats_history_buffer_size",
+ {offsetof(struct MutableDBOptions, stats_history_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_open_files",
+ {offsetof(struct MutableDBOptions, max_open_files), OptionType::kInt,
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
+ {"bytes_per_sync",
+ {offsetof(struct MutableDBOptions, bytes_per_sync),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"wal_bytes_per_sync",
+ {offsetof(struct MutableDBOptions, wal_bytes_per_sync),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"strict_bytes_per_sync",
+ {offsetof(struct MutableDBOptions, strict_bytes_per_sync),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"compaction_readahead_size",
+ {offsetof(struct MutableDBOptions, compaction_readahead_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+ {"max_background_flushes",
+ {offsetof(struct MutableDBOptions, max_background_flushes),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable}},
+};
+
+static std::unordered_map<std::string, OptionTypeInfo>
+ db_immutable_options_type_info = {
+ /*
+ // not yet supported
+ std::shared_ptr<Cache> row_cache;
+ std::shared_ptr<DeleteScheduler> delete_scheduler;
+ std::shared_ptr<Logger> info_log;
+ std::shared_ptr<RateLimiter> rate_limiter;
+ std::shared_ptr<Statistics> statistics;
+ std::vector<DbPath> db_paths;
+ FileTypeSet checksum_handoff_file_types;
+ */
+ {"advise_random_on_open",
+ {offsetof(struct ImmutableDBOptions, advise_random_on_open),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"allow_mmap_reads",
+ {offsetof(struct ImmutableDBOptions, allow_mmap_reads),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"allow_fallocate",
+ {offsetof(struct ImmutableDBOptions, allow_fallocate),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"allow_mmap_writes",
+ {offsetof(struct ImmutableDBOptions, allow_mmap_writes),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"use_direct_reads",
+ {offsetof(struct ImmutableDBOptions, use_direct_reads),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"use_direct_writes",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"use_direct_io_for_flush_and_compaction",
+ {offsetof(struct ImmutableDBOptions,
+ use_direct_io_for_flush_and_compaction),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"allow_2pc",
+ {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"wal_filter",
+ OptionTypeInfo::AsCustomRawPtr<WalFilter>(
+ offsetof(struct ImmutableDBOptions, wal_filter),
+ OptionVerificationType::kByName,
+ (OptionTypeFlags::kAllowNull | OptionTypeFlags::kCompareNever))},
+ {"create_if_missing",
+ {offsetof(struct ImmutableDBOptions, create_if_missing),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"create_missing_column_families",
+ {offsetof(struct ImmutableDBOptions, create_missing_column_families),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"disableDataSync",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"disable_data_sync", // for compatibility
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"enable_thread_tracking",
+ {offsetof(struct ImmutableDBOptions, enable_thread_tracking),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"error_if_exists",
+ {offsetof(struct ImmutableDBOptions, error_if_exists),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"experimental_allow_mempurge",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"experimental_mempurge_policy",
+ {0, OptionType::kString, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"experimental_mempurge_threshold",
+ {0, OptionType::kDouble, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"is_fd_close_on_exec",
+ {offsetof(struct ImmutableDBOptions, is_fd_close_on_exec),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"paranoid_checks",
+ {offsetof(struct ImmutableDBOptions, paranoid_checks),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"flush_verify_memtable_count",
+ {offsetof(struct ImmutableDBOptions, flush_verify_memtable_count),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"track_and_verify_wals_in_manifest",
+ {offsetof(struct ImmutableDBOptions,
+ track_and_verify_wals_in_manifest),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"verify_sst_unique_id_in_manifest",
+ {offsetof(struct ImmutableDBOptions, verify_sst_unique_id_in_manifest),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"skip_log_error_on_recovery",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"skip_stats_update_on_db_open",
+ {offsetof(struct ImmutableDBOptions, skip_stats_update_on_db_open),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"skip_checking_sst_file_sizes_on_db_open",
+ {offsetof(struct ImmutableDBOptions,
+ skip_checking_sst_file_sizes_on_db_open),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"new_table_reader_for_compaction_inputs",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"random_access_max_buffer_size",
+ {offsetof(struct ImmutableDBOptions, random_access_max_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"use_adaptive_mutex",
+ {offsetof(struct ImmutableDBOptions, use_adaptive_mutex),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"use_fsync",
+ {offsetof(struct ImmutableDBOptions, use_fsync), OptionType::kBoolean,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"max_file_opening_threads",
+ {offsetof(struct ImmutableDBOptions, max_file_opening_threads),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"table_cache_numshardbits",
+ {offsetof(struct ImmutableDBOptions, table_cache_numshardbits),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"db_write_buffer_size",
+ {offsetof(struct ImmutableDBOptions, db_write_buffer_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"keep_log_file_num",
+ {offsetof(struct ImmutableDBOptions, keep_log_file_num),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"recycle_log_file_num",
+ {offsetof(struct ImmutableDBOptions, recycle_log_file_num),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"log_file_time_to_roll",
+ {offsetof(struct ImmutableDBOptions, log_file_time_to_roll),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"manifest_preallocation_size",
+ {offsetof(struct ImmutableDBOptions, manifest_preallocation_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"max_log_file_size",
+ {offsetof(struct ImmutableDBOptions, max_log_file_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"db_log_dir",
+ {offsetof(struct ImmutableDBOptions, db_log_dir), OptionType::kString,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"wal_dir",
+ {offsetof(struct ImmutableDBOptions, wal_dir), OptionType::kString,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"WAL_size_limit_MB",
+ {offsetof(struct ImmutableDBOptions, WAL_size_limit_MB),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"WAL_ttl_seconds",
+ {offsetof(struct ImmutableDBOptions, WAL_ttl_seconds),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"max_manifest_file_size",
+ {offsetof(struct ImmutableDBOptions, max_manifest_file_size),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"persist_stats_to_disk",
+ {offsetof(struct ImmutableDBOptions, persist_stats_to_disk),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"fail_if_options_file_error",
+ {offsetof(struct ImmutableDBOptions, fail_if_options_file_error),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"enable_pipelined_write",
+ {offsetof(struct ImmutableDBOptions, enable_pipelined_write),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"unordered_write",
+ {offsetof(struct ImmutableDBOptions, unordered_write),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"allow_concurrent_memtable_write",
+ {offsetof(struct ImmutableDBOptions, allow_concurrent_memtable_write),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"wal_recovery_mode",
+ OptionTypeInfo::Enum<WALRecoveryMode>(
+ offsetof(struct ImmutableDBOptions, wal_recovery_mode),
+ &wal_recovery_mode_string_map)},
+ {"enable_write_thread_adaptive_yield",
+ {offsetof(struct ImmutableDBOptions,
+ enable_write_thread_adaptive_yield),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"write_thread_slow_yield_usec",
+ {offsetof(struct ImmutableDBOptions, write_thread_slow_yield_usec),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"max_write_batch_group_size_bytes",
+ {offsetof(struct ImmutableDBOptions, max_write_batch_group_size_bytes),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"write_thread_max_yield_usec",
+ {offsetof(struct ImmutableDBOptions, write_thread_max_yield_usec),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"access_hint_on_compaction_start",
+ OptionTypeInfo::Enum<DBOptions::AccessHint>(
+ offsetof(struct ImmutableDBOptions,
+ access_hint_on_compaction_start),
+ &access_hint_string_map)},
+ {"info_log_level",
+ OptionTypeInfo::Enum<InfoLogLevel>(
+ offsetof(struct ImmutableDBOptions, info_log_level),
+ &info_log_level_string_map)},
+ {"dump_malloc_stats",
+ {offsetof(struct ImmutableDBOptions, dump_malloc_stats),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"avoid_flush_during_recovery",
+ {offsetof(struct ImmutableDBOptions, avoid_flush_during_recovery),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"allow_ingest_behind",
+ {offsetof(struct ImmutableDBOptions, allow_ingest_behind),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"preserve_deletes",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"concurrent_prepare", // Deprecated by two_write_queues
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"two_write_queues",
+ {offsetof(struct ImmutableDBOptions, two_write_queues),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"manual_wal_flush",
+ {offsetof(struct ImmutableDBOptions, manual_wal_flush),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"wal_compression",
+ {offsetof(struct ImmutableDBOptions, wal_compression),
+ OptionType::kCompressionType, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"seq_per_batch",
+ {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone}},
+ {"atomic_flush",
+ {offsetof(struct ImmutableDBOptions, atomic_flush),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"avoid_unnecessary_blocking_io",
+ {offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"write_dbid_to_manifest",
+ {offsetof(struct ImmutableDBOptions, write_dbid_to_manifest),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"log_readahead_size",
+ {offsetof(struct ImmutableDBOptions, log_readahead_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"best_efforts_recovery",
+ {offsetof(struct ImmutableDBOptions, best_efforts_recovery),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"max_bgerror_resume_count",
+ {offsetof(struct ImmutableDBOptions, max_bgerror_resume_count),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"bgerror_resume_retry_interval",
+ {offsetof(struct ImmutableDBOptions, bgerror_resume_retry_interval),
+ OptionType::kUInt64T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"db_host_id",
+ {offsetof(struct ImmutableDBOptions, db_host_id), OptionType::kString,
+ OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}},
+ // Temporarily deprecated due to race conditions (examples in PR 10375).
+ {"rate_limiter",
+ {offsetof(struct ImmutableDBOptions, rate_limiter),
+ OptionType::kUnknown, OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever}},
+ // The following properties were handled as special cases in ParseOption
+ // This means that the properties could be read from the options file
+ // but never written to the file or compared to each other.
+ {"rate_limiter_bytes_per_sec",
+ {offsetof(struct ImmutableDBOptions, rate_limiter),
+ OptionType::kUnknown, OptionVerificationType::kNormal,
+ (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever),
+ // Parse the input value as a RateLimiter
+ [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
+ const std::string& value, void* addr) {
+ auto limiter = static_cast<std::shared_ptr<RateLimiter>*>(addr);
+ limiter->reset(NewGenericRateLimiter(
+ static_cast<int64_t>(ParseUint64(value))));
+ return Status::OK();
+ }}},
+ {"env", //**TODO: Should this be kCustomizable?
+ OptionTypeInfo(
+ offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown,
+ OptionVerificationType::kNormal,
+ (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever))
+ .SetParseFunc([](const ConfigOptions& opts,
+ const std::string& /*name*/,
+ const std::string& value, void* addr) {
+ // Parse the input value as an Env
+ auto old_env = static_cast<Env**>(addr); // Get the old value
+ Env* new_env = *old_env; // Set new to old
+ Status s = Env::CreateFromString(opts, value,
+ &new_env); // Update new value
+ if (s.ok()) { // It worked
+ *old_env = new_env; // Update the old one
+ }
+ return s;
+ })
+ .SetPrepareFunc([](const ConfigOptions& opts,
+ const std::string& /*name*/, void* addr) {
+ auto env = static_cast<Env**>(addr);
+ return (*env)->PrepareOptions(opts);
+ })
+ .SetValidateFunc([](const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts,
+ const std::string& /*name*/,
+ const void* addr) {
+ const auto env = static_cast<const Env* const*>(addr);
+ return (*env)->ValidateOptions(db_opts, cf_opts);
+ })},
+ {"allow_data_in_errors",
+ {offsetof(struct ImmutableDBOptions, allow_data_in_errors),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"file_checksum_gen_factory",
+ OptionTypeInfo::AsCustomSharedPtr<FileChecksumGenFactory>(
+ offsetof(struct ImmutableDBOptions, file_checksum_gen_factory),
+ OptionVerificationType::kByNameAllowFromNull,
+ OptionTypeFlags::kAllowNull)},
+ {"statistics",
+ OptionTypeInfo::AsCustomSharedPtr<Statistics>(
+ // Statistics should not be compared and can be null
+ // Statistics are maked "don't serialize" until they can be shared
+ // between DBs
+ offsetof(struct ImmutableDBOptions, statistics),
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize |
+ OptionTypeFlags::kAllowNull)},
+ // Allow EventListeners that have a non-empty Name() to be read/written
+ // as options Each listener will either be
+ // - A simple name (e.g. "MyEventListener")
+ // - A name with properties (e.g. "{id=MyListener1; timeout=60}"
+ // Multiple listeners will be separated by a ":":
+ // - "MyListener0;{id=MyListener1; timeout=60}
+ {"listeners",
+ {offsetof(struct ImmutableDBOptions, listeners), OptionType::kVector,
+ OptionVerificationType::kByNameAllowNull,
+ OptionTypeFlags::kCompareNever,
+ [](const ConfigOptions& opts, const std::string& /*name*/,
+ const std::string& value, void* addr) {
+ ConfigOptions embedded = opts;
+ embedded.ignore_unsupported_options = true;
+ std::vector<std::shared_ptr<EventListener>> listeners;
+ Status s;
+ for (size_t start = 0, end = 0;
+ s.ok() && start < value.size() && end != std::string::npos;
+ start = end + 1) {
+ std::string token;
+ s = OptionTypeInfo::NextToken(value, ':', start, &end, &token);
+ if (s.ok() && !token.empty()) {
+ std::shared_ptr<EventListener> listener;
+ s = EventListener::CreateFromString(embedded, token, &listener);
+ if (s.ok() && listener != nullptr) {
+ listeners.push_back(listener);
+ }
+ }
+ }
+ if (s.ok()) { // It worked
+ *(static_cast<std::vector<std::shared_ptr<EventListener>>*>(
+ addr)) = listeners;
+ }
+ return s;
+ },
+ [](const ConfigOptions& opts, const std::string& /*name*/,
+ const void* addr, std::string* value) {
+ const auto listeners =
+ static_cast<const std::vector<std::shared_ptr<EventListener>>*>(
+ addr);
+ ConfigOptions embedded = opts;
+ embedded.delimiter = ";";
+ int printed = 0;
+ for (const auto& listener : *listeners) {
+ auto id = listener->GetId();
+ if (!id.empty()) {
+ std::string elem_str = listener->ToString(embedded, "");
+ if (printed++ == 0) {
+ value->append("{");
+ } else {
+ value->append(":");
+ }
+ value->append(elem_str);
+ }
+ }
+ if (printed > 0) {
+ value->append("}");
+ }
+ return Status::OK();
+ },
+ nullptr}},
+ {"lowest_used_cache_tier",
+ OptionTypeInfo::Enum<CacheTier>(
+ offsetof(struct ImmutableDBOptions, lowest_used_cache_tier),
+ &cache_tier_string_map, OptionTypeFlags::kNone)},
+ {"enforce_single_del_contracts",
+ {offsetof(struct ImmutableDBOptions, enforce_single_del_contracts),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+};
+
+const std::string OptionsHelper::kDBOptionsName = "DBOptions";
+
+class MutableDBConfigurable : public Configurable {
+ public:
+ explicit MutableDBConfigurable(
+ const MutableDBOptions& mdb,
+ const std::unordered_map<std::string, std::string>* map = nullptr)
+ : mutable_(mdb), opt_map_(map) {
+ RegisterOptions(&mutable_, &db_mutable_options_type_info);
+ }
+
+ bool OptionsAreEqual(const ConfigOptions& config_options,
+ const OptionTypeInfo& opt_info,
+ const std::string& opt_name, const void* const this_ptr,
+ const void* const that_ptr,
+ std::string* mismatch) const override {
+ bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr,
+ that_ptr, mismatch);
+ if (!equals && opt_info.IsByName()) {
+ if (opt_map_ == nullptr) {
+ equals = true;
+ } else {
+ const auto& iter = opt_map_->find(opt_name);
+ if (iter == opt_map_->end()) {
+ equals = true;
+ } else {
+ equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr,
+ iter->second);
+ }
+ }
+ if (equals) { // False alarm, clear mismatch
+ *mismatch = "";
+ }
+ }
+ if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) {
+ const auto* this_config = opt_info.AsRawPointer<Configurable>(this_ptr);
+ if (this_config == nullptr) {
+ const auto& iter = opt_map_->find(opt_name);
+ // If the name exists in the map and is not empty/null,
+ // then the this_config should be set.
+ if (iter != opt_map_->end() && !iter->second.empty() &&
+ iter->second != kNullptrString) {
+ *mismatch = opt_name;
+ equals = false;
+ }
+ }
+ }
+ return equals;
+ }
+
+ protected:
+ MutableDBOptions mutable_;
+ const std::unordered_map<std::string, std::string>* opt_map_;
+};
+
+class DBOptionsConfigurable : public MutableDBConfigurable {
+ public:
+ explicit DBOptionsConfigurable(
+ const DBOptions& opts,
+ const std::unordered_map<std::string, std::string>* map = nullptr)
+ : MutableDBConfigurable(MutableDBOptions(opts), map), db_options_(opts) {
+ // The ImmutableDBOptions currently requires the env to be non-null. Make
+ // sure it is
+ if (opts.env != nullptr) {
+ immutable_ = ImmutableDBOptions(opts);
+ } else {
+ DBOptions copy = opts;
+ copy.env = Env::Default();
+ immutable_ = ImmutableDBOptions(copy);
+ }
+ RegisterOptions(&immutable_, &db_immutable_options_type_info);
+ }
+
+ protected:
+ Status ConfigureOptions(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ std::unordered_map<std::string, std::string>* unused) override {
+ Status s = Configurable::ConfigureOptions(config_options, opts_map, unused);
+ if (s.ok()) {
+ db_options_ = BuildDBOptions(immutable_, mutable_);
+ s = PrepareOptions(config_options);
+ }
+ return s;
+ }
+
+ const void* GetOptionsPtr(const std::string& name) const override {
+ if (name == OptionsHelper::kDBOptionsName) {
+ return &db_options_;
+ } else {
+ return MutableDBConfigurable::GetOptionsPtr(name);
+ }
+ }
+
+ private:
+ ImmutableDBOptions immutable_;
+ DBOptions db_options_;
+};
+
+std::unique_ptr<Configurable> DBOptionsAsConfigurable(
+ const MutableDBOptions& opts) {
+ std::unique_ptr<Configurable> ptr(new MutableDBConfigurable(opts));
+ return ptr;
+}
+std::unique_ptr<Configurable> DBOptionsAsConfigurable(
+ const DBOptions& opts,
+ const std::unordered_map<std::string, std::string>* opt_map) {
+ std::unique_ptr<Configurable> ptr(new DBOptionsConfigurable(opts, opt_map));
+ return ptr;
+}
+#endif // ROCKSDB_LITE
+
+ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {}
+
+ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
+ : create_if_missing(options.create_if_missing),
+ create_missing_column_families(options.create_missing_column_families),
+ error_if_exists(options.error_if_exists),
+ paranoid_checks(options.paranoid_checks),
+ flush_verify_memtable_count(options.flush_verify_memtable_count),
+ track_and_verify_wals_in_manifest(
+ options.track_and_verify_wals_in_manifest),
+ verify_sst_unique_id_in_manifest(
+ options.verify_sst_unique_id_in_manifest),
+ env(options.env),
+ rate_limiter(options.rate_limiter),
+ sst_file_manager(options.sst_file_manager),
+ info_log(options.info_log),
+ info_log_level(options.info_log_level),
+ max_file_opening_threads(options.max_file_opening_threads),
+ statistics(options.statistics),
+ use_fsync(options.use_fsync),
+ db_paths(options.db_paths),
+ db_log_dir(options.db_log_dir),
+ wal_dir(options.wal_dir),
+ max_log_file_size(options.max_log_file_size),
+ log_file_time_to_roll(options.log_file_time_to_roll),
+ keep_log_file_num(options.keep_log_file_num),
+ recycle_log_file_num(options.recycle_log_file_num),
+ max_manifest_file_size(options.max_manifest_file_size),
+ table_cache_numshardbits(options.table_cache_numshardbits),
+ WAL_ttl_seconds(options.WAL_ttl_seconds),
+ WAL_size_limit_MB(options.WAL_size_limit_MB),
+ max_write_batch_group_size_bytes(
+ options.max_write_batch_group_size_bytes),
+ manifest_preallocation_size(options.manifest_preallocation_size),
+ allow_mmap_reads(options.allow_mmap_reads),
+ allow_mmap_writes(options.allow_mmap_writes),
+ use_direct_reads(options.use_direct_reads),
+ use_direct_io_for_flush_and_compaction(
+ options.use_direct_io_for_flush_and_compaction),
+ allow_fallocate(options.allow_fallocate),
+ is_fd_close_on_exec(options.is_fd_close_on_exec),
+ advise_random_on_open(options.advise_random_on_open),
+ db_write_buffer_size(options.db_write_buffer_size),
+ write_buffer_manager(options.write_buffer_manager),
+ access_hint_on_compaction_start(options.access_hint_on_compaction_start),
+ random_access_max_buffer_size(options.random_access_max_buffer_size),
+ use_adaptive_mutex(options.use_adaptive_mutex),
+ listeners(options.listeners),
+ enable_thread_tracking(options.enable_thread_tracking),
+ enable_pipelined_write(options.enable_pipelined_write),
+ unordered_write(options.unordered_write),
+ allow_concurrent_memtable_write(options.allow_concurrent_memtable_write),
+ enable_write_thread_adaptive_yield(
+ options.enable_write_thread_adaptive_yield),
+ write_thread_max_yield_usec(options.write_thread_max_yield_usec),
+ write_thread_slow_yield_usec(options.write_thread_slow_yield_usec),
+ skip_stats_update_on_db_open(options.skip_stats_update_on_db_open),
+ skip_checking_sst_file_sizes_on_db_open(
+ options.skip_checking_sst_file_sizes_on_db_open),
+ wal_recovery_mode(options.wal_recovery_mode),
+ allow_2pc(options.allow_2pc),
+ row_cache(options.row_cache),
+#ifndef ROCKSDB_LITE
+ wal_filter(options.wal_filter),
+#endif // ROCKSDB_LITE
+ fail_if_options_file_error(options.fail_if_options_file_error),
+ dump_malloc_stats(options.dump_malloc_stats),
+ avoid_flush_during_recovery(options.avoid_flush_during_recovery),
+ allow_ingest_behind(options.allow_ingest_behind),
+ two_write_queues(options.two_write_queues),
+ manual_wal_flush(options.manual_wal_flush),
+ wal_compression(options.wal_compression),
+ atomic_flush(options.atomic_flush),
+ avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io),
+ persist_stats_to_disk(options.persist_stats_to_disk),
+ write_dbid_to_manifest(options.write_dbid_to_manifest),
+ log_readahead_size(options.log_readahead_size),
+ file_checksum_gen_factory(options.file_checksum_gen_factory),
+ best_efforts_recovery(options.best_efforts_recovery),
+ max_bgerror_resume_count(options.max_bgerror_resume_count),
+ bgerror_resume_retry_interval(options.bgerror_resume_retry_interval),
+ allow_data_in_errors(options.allow_data_in_errors),
+ db_host_id(options.db_host_id),
+ checksum_handoff_file_types(options.checksum_handoff_file_types),
+ lowest_used_cache_tier(options.lowest_used_cache_tier),
+ compaction_service(options.compaction_service),
+ enforce_single_del_contracts(options.enforce_single_del_contracts) {
+ fs = env->GetFileSystem();
+ clock = env->GetSystemClock().get();
+ logger = info_log.get();
+ stats = statistics.get();
+}
+
+void ImmutableDBOptions::Dump(Logger* log) const {
+ ROCKS_LOG_HEADER(log, " Options.error_if_exists: %d",
+ error_if_exists);
+ ROCKS_LOG_HEADER(log, " Options.create_if_missing: %d",
+ create_if_missing);
+ ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d",
+ paranoid_checks);
+ ROCKS_LOG_HEADER(log, " Options.flush_verify_memtable_count: %d",
+ flush_verify_memtable_count);
+ ROCKS_LOG_HEADER(log,
+ " "
+ "Options.track_and_verify_wals_in_manifest: %d",
+ track_and_verify_wals_in_manifest);
+ ROCKS_LOG_HEADER(log, " Options.verify_sst_unique_id_in_manifest: %d",
+ verify_sst_unique_id_in_manifest);
+ ROCKS_LOG_HEADER(log, " Options.env: %p",
+ env);
+ ROCKS_LOG_HEADER(log, " Options.fs: %s",
+ fs->Name());
+ ROCKS_LOG_HEADER(log, " Options.info_log: %p",
+ info_log.get());
+ ROCKS_LOG_HEADER(log, " Options.max_file_opening_threads: %d",
+ max_file_opening_threads);
+ ROCKS_LOG_HEADER(log, " Options.statistics: %p",
+ stats);
+ ROCKS_LOG_HEADER(log, " Options.use_fsync: %d",
+ use_fsync);
+ ROCKS_LOG_HEADER(
+ log, " Options.max_log_file_size: %" ROCKSDB_PRIszt,
+ max_log_file_size);
+ ROCKS_LOG_HEADER(log,
+ " Options.max_manifest_file_size: %" PRIu64,
+ max_manifest_file_size);
+ ROCKS_LOG_HEADER(
+ log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt,
+ log_file_time_to_roll);
+ ROCKS_LOG_HEADER(
+ log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt,
+ keep_log_file_num);
+ ROCKS_LOG_HEADER(
+ log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt,
+ recycle_log_file_num);
+ ROCKS_LOG_HEADER(log, " Options.allow_fallocate: %d",
+ allow_fallocate);
+ ROCKS_LOG_HEADER(log, " Options.allow_mmap_reads: %d",
+ allow_mmap_reads);
+ ROCKS_LOG_HEADER(log, " Options.allow_mmap_writes: %d",
+ allow_mmap_writes);
+ ROCKS_LOG_HEADER(log, " Options.use_direct_reads: %d",
+ use_direct_reads);
+ ROCKS_LOG_HEADER(log,
+ " "
+ "Options.use_direct_io_for_flush_and_compaction: %d",
+ use_direct_io_for_flush_and_compaction);
+ ROCKS_LOG_HEADER(log, " Options.create_missing_column_families: %d",
+ create_missing_column_families);
+ ROCKS_LOG_HEADER(log, " Options.db_log_dir: %s",
+ db_log_dir.c_str());
+ ROCKS_LOG_HEADER(log, " Options.wal_dir: %s",
+ wal_dir.c_str());
+ ROCKS_LOG_HEADER(log, " Options.table_cache_numshardbits: %d",
+ table_cache_numshardbits);
+ ROCKS_LOG_HEADER(log,
+ " Options.WAL_ttl_seconds: %" PRIu64,
+ WAL_ttl_seconds);
+ ROCKS_LOG_HEADER(log,
+ " Options.WAL_size_limit_MB: %" PRIu64,
+ WAL_size_limit_MB);
+ ROCKS_LOG_HEADER(log,
+ " "
+ "Options.max_write_batch_group_size_bytes: %" PRIu64,
+ max_write_batch_group_size_bytes);
+ ROCKS_LOG_HEADER(
+ log, " Options.manifest_preallocation_size: %" ROCKSDB_PRIszt,
+ manifest_preallocation_size);
+ ROCKS_LOG_HEADER(log, " Options.is_fd_close_on_exec: %d",
+ is_fd_close_on_exec);
+ ROCKS_LOG_HEADER(log, " Options.advise_random_on_open: %d",
+ advise_random_on_open);
+ ROCKS_LOG_HEADER(
+ log, " Options.db_write_buffer_size: %" ROCKSDB_PRIszt,
+ db_write_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.write_buffer_manager: %p",
+ write_buffer_manager.get());
+ ROCKS_LOG_HEADER(log, " Options.access_hint_on_compaction_start: %d",
+ static_cast<int>(access_hint_on_compaction_start));
+ ROCKS_LOG_HEADER(
+ log, " Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt,
+ random_access_max_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.use_adaptive_mutex: %d",
+ use_adaptive_mutex);
+ ROCKS_LOG_HEADER(log, " Options.rate_limiter: %p",
+ rate_limiter.get());
+ Header(
+ log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64,
+ sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0);
+ ROCKS_LOG_HEADER(log, " Options.wal_recovery_mode: %d",
+ static_cast<int>(wal_recovery_mode));
+ ROCKS_LOG_HEADER(log, " Options.enable_thread_tracking: %d",
+ enable_thread_tracking);
+ ROCKS_LOG_HEADER(log, " Options.enable_pipelined_write: %d",
+ enable_pipelined_write);
+ ROCKS_LOG_HEADER(log, " Options.unordered_write: %d",
+ unordered_write);
+ ROCKS_LOG_HEADER(log, " Options.allow_concurrent_memtable_write: %d",
+ allow_concurrent_memtable_write);
+ ROCKS_LOG_HEADER(log, " Options.enable_write_thread_adaptive_yield: %d",
+ enable_write_thread_adaptive_yield);
+ ROCKS_LOG_HEADER(log,
+ " Options.write_thread_max_yield_usec: %" PRIu64,
+ write_thread_max_yield_usec);
+ ROCKS_LOG_HEADER(log,
+ " Options.write_thread_slow_yield_usec: %" PRIu64,
+ write_thread_slow_yield_usec);
+ if (row_cache) {
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.row_cache: %" ROCKSDB_PRIszt,
+ row_cache->GetCapacity());
+ } else {
+ ROCKS_LOG_HEADER(log,
+ " Options.row_cache: None");
+ }
+#ifndef ROCKSDB_LITE
+ ROCKS_LOG_HEADER(log, " Options.wal_filter: %s",
+ wal_filter ? wal_filter->Name() : "None");
+#endif // ROCKDB_LITE
+
+ ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_recovery: %d",
+ avoid_flush_during_recovery);
+ ROCKS_LOG_HEADER(log, " Options.allow_ingest_behind: %d",
+ allow_ingest_behind);
+ ROCKS_LOG_HEADER(log, " Options.two_write_queues: %d",
+ two_write_queues);
+ ROCKS_LOG_HEADER(log, " Options.manual_wal_flush: %d",
+ manual_wal_flush);
+ ROCKS_LOG_HEADER(log, " Options.wal_compression: %d",
+ wal_compression);
+ ROCKS_LOG_HEADER(log, " Options.atomic_flush: %d", atomic_flush);
+ ROCKS_LOG_HEADER(log,
+ " Options.avoid_unnecessary_blocking_io: %d",
+ avoid_unnecessary_blocking_io);
+ ROCKS_LOG_HEADER(log, " Options.persist_stats_to_disk: %u",
+ persist_stats_to_disk);
+ ROCKS_LOG_HEADER(log, " Options.write_dbid_to_manifest: %d",
+ write_dbid_to_manifest);
+ ROCKS_LOG_HEADER(
+ log, " Options.log_readahead_size: %" ROCKSDB_PRIszt,
+ log_readahead_size);
+ ROCKS_LOG_HEADER(log, " Options.file_checksum_gen_factory: %s",
+ file_checksum_gen_factory ? file_checksum_gen_factory->Name()
+ : kUnknownFileChecksumFuncName);
+ ROCKS_LOG_HEADER(log, " Options.best_efforts_recovery: %d",
+ static_cast<int>(best_efforts_recovery));
+ ROCKS_LOG_HEADER(log, " Options.max_bgerror_resume_count: %d",
+ max_bgerror_resume_count);
+ ROCKS_LOG_HEADER(log,
+ " Options.bgerror_resume_retry_interval: %" PRIu64,
+ bgerror_resume_retry_interval);
+ ROCKS_LOG_HEADER(log, " Options.allow_data_in_errors: %d",
+ allow_data_in_errors);
+ ROCKS_LOG_HEADER(log, " Options.db_host_id: %s",
+ db_host_id.c_str());
+ ROCKS_LOG_HEADER(log, " Options.enforce_single_del_contracts: %s",
+ enforce_single_del_contracts ? "true" : "false");
+}
+
+bool ImmutableDBOptions::IsWalDirSameAsDBPath() const {
+ assert(!db_paths.empty());
+ return IsWalDirSameAsDBPath(db_paths[0].path);
+}
+
+bool ImmutableDBOptions::IsWalDirSameAsDBPath(
+ const std::string& db_path) const {
+ bool same = wal_dir.empty();
+ if (!same) {
+ Status s = env->AreFilesSame(wal_dir, db_path, &same);
+ if (s.IsNotSupported()) {
+ same = wal_dir == db_path;
+ }
+ }
+ return same;
+}
+
+const std::string& ImmutableDBOptions::GetWalDir() const {
+ if (wal_dir.empty()) {
+ assert(!db_paths.empty());
+ return db_paths[0].path;
+ } else {
+ return wal_dir;
+ }
+}
+
+const std::string& ImmutableDBOptions::GetWalDir(
+ const std::string& path) const {
+ if (wal_dir.empty()) {
+ return path;
+ } else {
+ return wal_dir;
+ }
+}
+
+MutableDBOptions::MutableDBOptions()
+ : max_background_jobs(2),
+ max_background_compactions(-1),
+ max_subcompactions(0),
+ avoid_flush_during_shutdown(false),
+ writable_file_max_buffer_size(1024 * 1024),
+ delayed_write_rate(2 * 1024U * 1024U),
+ max_total_wal_size(0),
+ delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000),
+ stats_dump_period_sec(600),
+ stats_persist_period_sec(600),
+ stats_history_buffer_size(1024 * 1024),
+ max_open_files(-1),
+ bytes_per_sync(0),
+ wal_bytes_per_sync(0),
+ strict_bytes_per_sync(false),
+ compaction_readahead_size(0),
+ max_background_flushes(-1) {}
+
+MutableDBOptions::MutableDBOptions(const DBOptions& options)
+ : max_background_jobs(options.max_background_jobs),
+ max_background_compactions(options.max_background_compactions),
+ max_subcompactions(options.max_subcompactions),
+ avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
+ writable_file_max_buffer_size(options.writable_file_max_buffer_size),
+ delayed_write_rate(options.delayed_write_rate),
+ max_total_wal_size(options.max_total_wal_size),
+ delete_obsolete_files_period_micros(
+ options.delete_obsolete_files_period_micros),
+ stats_dump_period_sec(options.stats_dump_period_sec),
+ stats_persist_period_sec(options.stats_persist_period_sec),
+ stats_history_buffer_size(options.stats_history_buffer_size),
+ max_open_files(options.max_open_files),
+ bytes_per_sync(options.bytes_per_sync),
+ wal_bytes_per_sync(options.wal_bytes_per_sync),
+ strict_bytes_per_sync(options.strict_bytes_per_sync),
+ compaction_readahead_size(options.compaction_readahead_size),
+ max_background_flushes(options.max_background_flushes) {}
+
+void MutableDBOptions::Dump(Logger* log) const {
+ ROCKS_LOG_HEADER(log, " Options.max_background_jobs: %d",
+ max_background_jobs);
+ ROCKS_LOG_HEADER(log, " Options.max_background_compactions: %d",
+ max_background_compactions);
+ ROCKS_LOG_HEADER(log, " Options.max_subcompactions: %" PRIu32,
+ max_subcompactions);
+ ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_shutdown: %d",
+ avoid_flush_during_shutdown);
+ ROCKS_LOG_HEADER(
+ log, " Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt,
+ writable_file_max_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.delayed_write_rate : %" PRIu64,
+ delayed_write_rate);
+ ROCKS_LOG_HEADER(log, " Options.max_total_wal_size: %" PRIu64,
+ max_total_wal_size);
+ ROCKS_LOG_HEADER(
+ log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
+ delete_obsolete_files_period_micros);
+ ROCKS_LOG_HEADER(log, " Options.stats_dump_period_sec: %u",
+ stats_dump_period_sec);
+ ROCKS_LOG_HEADER(log, " Options.stats_persist_period_sec: %d",
+ stats_persist_period_sec);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.stats_history_buffer_size: %" ROCKSDB_PRIszt,
+ stats_history_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.max_open_files: %d",
+ max_open_files);
+ ROCKS_LOG_HEADER(log,
+ " Options.bytes_per_sync: %" PRIu64,
+ bytes_per_sync);
+ ROCKS_LOG_HEADER(log,
+ " Options.wal_bytes_per_sync: %" PRIu64,
+ wal_bytes_per_sync);
+ ROCKS_LOG_HEADER(log,
+ " Options.strict_bytes_per_sync: %d",
+ strict_bytes_per_sync);
+ ROCKS_LOG_HEADER(log,
+ " Options.compaction_readahead_size: %" ROCKSDB_PRIszt,
+ compaction_readahead_size);
+ ROCKS_LOG_HEADER(log, " Options.max_background_flushes: %d",
+ max_background_flushes);
+}
+
+#ifndef ROCKSDB_LITE
+Status GetMutableDBOptionsFromStrings(
+ const MutableDBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ MutableDBOptions* new_options) {
+ assert(new_options);
+ *new_options = base_options;
+ ConfigOptions config_options;
+ Status s = OptionTypeInfo::ParseType(
+ config_options, options_map, db_mutable_options_type_info, new_options);
+ if (!s.ok()) {
+ *new_options = base_options;
+ }
+ return s;
+}
+
+bool MutableDBOptionsAreEqual(const MutableDBOptions& this_options,
+ const MutableDBOptions& that_options) {
+ ConfigOptions config_options;
+ std::string mismatch;
+ return OptionTypeInfo::StructsAreEqual(
+ config_options, "MutableDBOptions", &db_mutable_options_type_info,
+ "MutableDBOptions", &this_options, &that_options, &mismatch);
+}
+
+Status GetStringFromMutableDBOptions(const ConfigOptions& config_options,
+ const MutableDBOptions& mutable_opts,
+ std::string* opt_string) {
+ return OptionTypeInfo::SerializeType(
+ config_options, db_mutable_options_type_info, &mutable_opts, opt_string);
+}
+#endif // ROCKSDB_LITE
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/db_options.h b/src/rocksdb/options/db_options.h
new file mode 100644
index 000000000..8946f60ff
--- /dev/null
+++ b/src/rocksdb/options/db_options.h
@@ -0,0 +1,156 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "rocksdb/options.h"
+
+namespace ROCKSDB_NAMESPACE {
+class SystemClock;
+
+struct ImmutableDBOptions {
+ static const char* kName() { return "ImmutableDBOptions"; }
+ ImmutableDBOptions();
+ explicit ImmutableDBOptions(const DBOptions& options);
+
+ void Dump(Logger* log) const;
+
+ bool create_if_missing;
+ bool create_missing_column_families;
+ bool error_if_exists;
+ bool paranoid_checks;
+ bool flush_verify_memtable_count;
+ bool track_and_verify_wals_in_manifest;
+ bool verify_sst_unique_id_in_manifest;
+ Env* env;
+ std::shared_ptr<RateLimiter> rate_limiter;
+ std::shared_ptr<SstFileManager> sst_file_manager;
+ std::shared_ptr<Logger> info_log;
+ InfoLogLevel info_log_level;
+ int max_file_opening_threads;
+ std::shared_ptr<Statistics> statistics;
+ bool use_fsync;
+ std::vector<DbPath> db_paths;
+ std::string db_log_dir;
+ // The wal_dir option from the file. To determine the
+ // directory in use, the GetWalDir or IsWalDirSameAsDBPath
+ // methods should be used instead of accessing this variable directly.
+ std::string wal_dir;
+ size_t max_log_file_size;
+ size_t log_file_time_to_roll;
+ size_t keep_log_file_num;
+ size_t recycle_log_file_num;
+ uint64_t max_manifest_file_size;
+ int table_cache_numshardbits;
+ uint64_t WAL_ttl_seconds;
+ uint64_t WAL_size_limit_MB;
+ uint64_t max_write_batch_group_size_bytes;
+ size_t manifest_preallocation_size;
+ bool allow_mmap_reads;
+ bool allow_mmap_writes;
+ bool use_direct_reads;
+ bool use_direct_io_for_flush_and_compaction;
+ bool allow_fallocate;
+ bool is_fd_close_on_exec;
+ bool advise_random_on_open;
+ size_t db_write_buffer_size;
+ std::shared_ptr<WriteBufferManager> write_buffer_manager;
+ DBOptions::AccessHint access_hint_on_compaction_start;
+ size_t random_access_max_buffer_size;
+ bool use_adaptive_mutex;
+ std::vector<std::shared_ptr<EventListener>> listeners;
+ bool enable_thread_tracking;
+ bool enable_pipelined_write;
+ bool unordered_write;
+ bool allow_concurrent_memtable_write;
+ bool enable_write_thread_adaptive_yield;
+ uint64_t write_thread_max_yield_usec;
+ uint64_t write_thread_slow_yield_usec;
+ bool skip_stats_update_on_db_open;
+ bool skip_checking_sst_file_sizes_on_db_open;
+ WALRecoveryMode wal_recovery_mode;
+ bool allow_2pc;
+ std::shared_ptr<Cache> row_cache;
+#ifndef ROCKSDB_LITE
+ WalFilter* wal_filter;
+#endif // ROCKSDB_LITE
+ bool fail_if_options_file_error;
+ bool dump_malloc_stats;
+ bool avoid_flush_during_recovery;
+ bool allow_ingest_behind;
+ bool two_write_queues;
+ bool manual_wal_flush;
+ CompressionType wal_compression;
+ bool atomic_flush;
+ bool avoid_unnecessary_blocking_io;
+ bool persist_stats_to_disk;
+ bool write_dbid_to_manifest;
+ size_t log_readahead_size;
+ std::shared_ptr<FileChecksumGenFactory> file_checksum_gen_factory;
+ bool best_efforts_recovery;
+ int max_bgerror_resume_count;
+ uint64_t bgerror_resume_retry_interval;
+ bool allow_data_in_errors;
+ std::string db_host_id;
+ FileTypeSet checksum_handoff_file_types;
+ CacheTier lowest_used_cache_tier;
+ // Convenience/Helper objects that are not part of the base DBOptions
+ std::shared_ptr<FileSystem> fs;
+ SystemClock* clock;
+ Statistics* stats;
+ Logger* logger;
+ std::shared_ptr<CompactionService> compaction_service;
+ bool enforce_single_del_contracts;
+
+ bool IsWalDirSameAsDBPath() const;
+ bool IsWalDirSameAsDBPath(const std::string& path) const;
+ const std::string& GetWalDir() const;
+ const std::string& GetWalDir(const std::string& path) const;
+};
+
+struct MutableDBOptions {
+ static const char* kName() { return "MutableDBOptions"; }
+ MutableDBOptions();
+ explicit MutableDBOptions(const DBOptions& options);
+
+ void Dump(Logger* log) const;
+
+ int max_background_jobs;
+ int max_background_compactions;
+ uint32_t max_subcompactions;
+ bool avoid_flush_during_shutdown;
+ size_t writable_file_max_buffer_size;
+ uint64_t delayed_write_rate;
+ uint64_t max_total_wal_size;
+ uint64_t delete_obsolete_files_period_micros;
+ unsigned int stats_dump_period_sec;
+ unsigned int stats_persist_period_sec;
+ size_t stats_history_buffer_size;
+ int max_open_files;
+ uint64_t bytes_per_sync;
+ uint64_t wal_bytes_per_sync;
+ bool strict_bytes_per_sync;
+ size_t compaction_readahead_size;
+ int max_background_flushes;
+};
+
+#ifndef ROCKSDB_LITE
+Status GetStringFromMutableDBOptions(const ConfigOptions& config_options,
+ const MutableDBOptions& mutable_opts,
+ std::string* opt_string);
+
+Status GetMutableDBOptionsFromStrings(
+ const MutableDBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& options_map,
+ MutableDBOptions* new_options);
+
+bool MutableDBOptionsAreEqual(const MutableDBOptions& this_options,
+ const MutableDBOptions& that_options);
+#endif // ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options.cc b/src/rocksdb/options/options.cc
new file mode 100644
index 000000000..316d3550e
--- /dev/null
+++ b/src/rocksdb/options/options.cc
@@ -0,0 +1,735 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "rocksdb/options.h"
+
+#include <cinttypes>
+#include <limits>
+
+#include "logging/logging.h"
+#include "monitoring/statistics.h"
+#include "options/db_options.h"
+#include "options/options_helper.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/sst_file_manager.h"
+#include "rocksdb/sst_partitioner.h"
+#include "rocksdb/table.h"
+#include "rocksdb/table_properties.h"
+#include "rocksdb/wal_filter.h"
+#include "table/block_based/block_based_table_factory.h"
+#include "util/compression.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions() {
+ assert(memtable_factory.get() != nullptr);
+}
+
+AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options)
+ : max_write_buffer_number(options.max_write_buffer_number),
+ min_write_buffer_number_to_merge(
+ options.min_write_buffer_number_to_merge),
+ max_write_buffer_number_to_maintain(
+ options.max_write_buffer_number_to_maintain),
+ max_write_buffer_size_to_maintain(
+ options.max_write_buffer_size_to_maintain),
+ inplace_update_support(options.inplace_update_support),
+ inplace_update_num_locks(options.inplace_update_num_locks),
+ experimental_mempurge_threshold(options.experimental_mempurge_threshold),
+ inplace_callback(options.inplace_callback),
+ memtable_prefix_bloom_size_ratio(
+ options.memtable_prefix_bloom_size_ratio),
+ memtable_whole_key_filtering(options.memtable_whole_key_filtering),
+ memtable_huge_page_size(options.memtable_huge_page_size),
+ memtable_insert_with_hint_prefix_extractor(
+ options.memtable_insert_with_hint_prefix_extractor),
+ bloom_locality(options.bloom_locality),
+ arena_block_size(options.arena_block_size),
+ compression_per_level(options.compression_per_level),
+ num_levels(options.num_levels),
+ level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger),
+ level0_stop_writes_trigger(options.level0_stop_writes_trigger),
+ target_file_size_base(options.target_file_size_base),
+ target_file_size_multiplier(options.target_file_size_multiplier),
+ level_compaction_dynamic_level_bytes(
+ options.level_compaction_dynamic_level_bytes),
+ max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
+ max_bytes_for_level_multiplier_additional(
+ options.max_bytes_for_level_multiplier_additional),
+ max_compaction_bytes(options.max_compaction_bytes),
+ ignore_max_compaction_bytes_for_input(
+ options.ignore_max_compaction_bytes_for_input),
+ soft_pending_compaction_bytes_limit(
+ options.soft_pending_compaction_bytes_limit),
+ hard_pending_compaction_bytes_limit(
+ options.hard_pending_compaction_bytes_limit),
+ compaction_style(options.compaction_style),
+ compaction_pri(options.compaction_pri),
+ compaction_options_universal(options.compaction_options_universal),
+ compaction_options_fifo(options.compaction_options_fifo),
+ max_sequential_skip_in_iterations(
+ options.max_sequential_skip_in_iterations),
+ memtable_factory(options.memtable_factory),
+ table_properties_collector_factories(
+ options.table_properties_collector_factories),
+ max_successive_merges(options.max_successive_merges),
+ optimize_filters_for_hits(options.optimize_filters_for_hits),
+ paranoid_file_checks(options.paranoid_file_checks),
+ force_consistency_checks(options.force_consistency_checks),
+ report_bg_io_stats(options.report_bg_io_stats),
+ ttl(options.ttl),
+ periodic_compaction_seconds(options.periodic_compaction_seconds),
+ sample_for_compression(options.sample_for_compression),
+ preclude_last_level_data_seconds(
+ options.preclude_last_level_data_seconds),
+ preserve_internal_time_seconds(options.preserve_internal_time_seconds),
+ enable_blob_files(options.enable_blob_files),
+ min_blob_size(options.min_blob_size),
+ blob_file_size(options.blob_file_size),
+ blob_compression_type(options.blob_compression_type),
+ enable_blob_garbage_collection(options.enable_blob_garbage_collection),
+ blob_garbage_collection_age_cutoff(
+ options.blob_garbage_collection_age_cutoff),
+ blob_garbage_collection_force_threshold(
+ options.blob_garbage_collection_force_threshold),
+ blob_compaction_readahead_size(options.blob_compaction_readahead_size),
+ blob_file_starting_level(options.blob_file_starting_level),
+ blob_cache(options.blob_cache),
+ prepopulate_blob_cache(options.prepopulate_blob_cache) {
+ assert(memtable_factory.get() != nullptr);
+ if (max_bytes_for_level_multiplier_additional.size() <
+ static_cast<unsigned int>(num_levels)) {
+ max_bytes_for_level_multiplier_additional.resize(num_levels, 1);
+ }
+}
+
+ColumnFamilyOptions::ColumnFamilyOptions()
+ : compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
+ table_factory(
+ std::shared_ptr<TableFactory>(new BlockBasedTableFactory())) {}
+
+ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
+ : ColumnFamilyOptions(*static_cast<const ColumnFamilyOptions*>(&options)) {}
+
+DBOptions::DBOptions() {}
+DBOptions::DBOptions(const Options& options)
+ : DBOptions(*static_cast<const DBOptions*>(&options)) {}
+
+void DBOptions::Dump(Logger* log) const {
+ ImmutableDBOptions(*this).Dump(log);
+ MutableDBOptions(*this).Dump(log);
+} // DBOptions::Dump
+
+void ColumnFamilyOptions::Dump(Logger* log) const {
+ ROCKS_LOG_HEADER(log, " Options.comparator: %s",
+ comparator->Name());
+ ROCKS_LOG_HEADER(log, " Options.merge_operator: %s",
+ merge_operator ? merge_operator->Name() : "None");
+ ROCKS_LOG_HEADER(log, " Options.compaction_filter: %s",
+ compaction_filter ? compaction_filter->Name() : "None");
+ ROCKS_LOG_HEADER(
+ log, " Options.compaction_filter_factory: %s",
+ compaction_filter_factory ? compaction_filter_factory->Name() : "None");
+ ROCKS_LOG_HEADER(
+ log, " Options.sst_partitioner_factory: %s",
+ sst_partitioner_factory ? sst_partitioner_factory->Name() : "None");
+ ROCKS_LOG_HEADER(log, " Options.memtable_factory: %s",
+ memtable_factory->Name());
+ ROCKS_LOG_HEADER(log, " Options.table_factory: %s",
+ table_factory->Name());
+ ROCKS_LOG_HEADER(log, " table_factory options: %s",
+ table_factory->GetPrintableOptions().c_str());
+ ROCKS_LOG_HEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt,
+ write_buffer_size);
+ ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number: %d",
+ max_write_buffer_number);
+ if (!compression_per_level.empty()) {
+ for (unsigned int i = 0; i < compression_per_level.size(); i++) {
+ ROCKS_LOG_HEADER(
+ log, " Options.compression[%d]: %s", i,
+ CompressionTypeToString(compression_per_level[i]).c_str());
+ }
+ } else {
+ ROCKS_LOG_HEADER(log, " Options.compression: %s",
+ CompressionTypeToString(compression).c_str());
+ }
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression: %s",
+ bottommost_compression == kDisableCompressionOption
+ ? "Disabled"
+ : CompressionTypeToString(bottommost_compression).c_str());
+ ROCKS_LOG_HEADER(
+ log, " Options.prefix_extractor: %s",
+ prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
+ ROCKS_LOG_HEADER(log,
+ " Options.memtable_insert_with_hint_prefix_extractor: %s",
+ memtable_insert_with_hint_prefix_extractor == nullptr
+ ? "nullptr"
+ : memtable_insert_with_hint_prefix_extractor->Name());
+ ROCKS_LOG_HEADER(log, " Options.num_levels: %d", num_levels);
+ ROCKS_LOG_HEADER(log, " Options.min_write_buffer_number_to_merge: %d",
+ min_write_buffer_number_to_merge);
+ ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number_to_maintain: %d",
+ max_write_buffer_number_to_maintain);
+ ROCKS_LOG_HEADER(log,
+ " Options.max_write_buffer_size_to_maintain: %" PRIu64,
+ max_write_buffer_size_to_maintain);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.window_bits: %d",
+ bottommost_compression_opts.window_bits);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.level: %d",
+ bottommost_compression_opts.level);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.strategy: %d",
+ bottommost_compression_opts.strategy);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.bottommost_compression_opts.max_dict_bytes: "
+ "%" PRIu32,
+ bottommost_compression_opts.max_dict_bytes);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.bottommost_compression_opts.zstd_max_train_bytes: "
+ "%" PRIu32,
+ bottommost_compression_opts.zstd_max_train_bytes);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.bottommost_compression_opts.parallel_threads: "
+ "%" PRIu32,
+ bottommost_compression_opts.parallel_threads);
+ ROCKS_LOG_HEADER(
+ log, " Options.bottommost_compression_opts.enabled: %s",
+ bottommost_compression_opts.enabled ? "true" : "false");
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.bottommost_compression_opts.max_dict_buffer_bytes: "
+ "%" PRIu64,
+ bottommost_compression_opts.max_dict_buffer_bytes);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.bottommost_compression_opts.use_zstd_dict_trainer: %s",
+ bottommost_compression_opts.use_zstd_dict_trainer ? "true" : "false");
+ ROCKS_LOG_HEADER(log, " Options.compression_opts.window_bits: %d",
+ compression_opts.window_bits);
+ ROCKS_LOG_HEADER(log, " Options.compression_opts.level: %d",
+ compression_opts.level);
+ ROCKS_LOG_HEADER(log, " Options.compression_opts.strategy: %d",
+ compression_opts.strategy);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.compression_opts.max_dict_bytes: %" PRIu32,
+ compression_opts.max_dict_bytes);
+ ROCKS_LOG_HEADER(log,
+ " Options.compression_opts.zstd_max_train_bytes: "
+ "%" PRIu32,
+ compression_opts.zstd_max_train_bytes);
+ ROCKS_LOG_HEADER(
+ log, " Options.compression_opts.use_zstd_dict_trainer: %s",
+ compression_opts.use_zstd_dict_trainer ? "true" : "false");
+ ROCKS_LOG_HEADER(log,
+ " Options.compression_opts.parallel_threads: "
+ "%" PRIu32,
+ compression_opts.parallel_threads);
+ ROCKS_LOG_HEADER(log,
+ " Options.compression_opts.enabled: %s",
+ compression_opts.enabled ? "true" : "false");
+ ROCKS_LOG_HEADER(log,
+ " Options.compression_opts.max_dict_buffer_bytes: "
+ "%" PRIu64,
+ compression_opts.max_dict_buffer_bytes);
+ ROCKS_LOG_HEADER(log, " Options.level0_file_num_compaction_trigger: %d",
+ level0_file_num_compaction_trigger);
+ ROCKS_LOG_HEADER(log, " Options.level0_slowdown_writes_trigger: %d",
+ level0_slowdown_writes_trigger);
+ ROCKS_LOG_HEADER(log, " Options.level0_stop_writes_trigger: %d",
+ level0_stop_writes_trigger);
+ ROCKS_LOG_HEADER(
+ log, " Options.target_file_size_base: %" PRIu64,
+ target_file_size_base);
+ ROCKS_LOG_HEADER(log, " Options.target_file_size_multiplier: %d",
+ target_file_size_multiplier);
+ ROCKS_LOG_HEADER(
+ log, " Options.max_bytes_for_level_base: %" PRIu64,
+ max_bytes_for_level_base);
+ ROCKS_LOG_HEADER(log, "Options.level_compaction_dynamic_level_bytes: %d",
+ level_compaction_dynamic_level_bytes);
+ ROCKS_LOG_HEADER(log, " Options.max_bytes_for_level_multiplier: %f",
+ max_bytes_for_level_multiplier);
+ for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size();
+ i++) {
+ ROCKS_LOG_HEADER(
+ log, "Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt
+ "]: %d",
+ i, max_bytes_for_level_multiplier_additional[i]);
+ }
+ ROCKS_LOG_HEADER(
+ log, " Options.max_sequential_skip_in_iterations: %" PRIu64,
+ max_sequential_skip_in_iterations);
+ ROCKS_LOG_HEADER(
+ log, " Options.max_compaction_bytes: %" PRIu64,
+ max_compaction_bytes);
+ ROCKS_LOG_HEADER(log, " Options.ignore_max_compaction_bytes_for_input: %s",
+ ignore_max_compaction_bytes_for_input ? "true" : "false");
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.arena_block_size: %" ROCKSDB_PRIszt,
+ arena_block_size);
+ ROCKS_LOG_HEADER(log,
+ " Options.soft_pending_compaction_bytes_limit: %" PRIu64,
+ soft_pending_compaction_bytes_limit);
+ ROCKS_LOG_HEADER(log,
+ " Options.hard_pending_compaction_bytes_limit: %" PRIu64,
+ hard_pending_compaction_bytes_limit);
+ ROCKS_LOG_HEADER(log, " Options.disable_auto_compactions: %d",
+ disable_auto_compactions);
+
+ const auto& it_compaction_style =
+ compaction_style_to_string.find(compaction_style);
+ std::string str_compaction_style;
+ if (it_compaction_style == compaction_style_to_string.end()) {
+ assert(false);
+ str_compaction_style = "unknown_" + std::to_string(compaction_style);
+ } else {
+ str_compaction_style = it_compaction_style->second;
+ }
+ ROCKS_LOG_HEADER(log,
+ " Options.compaction_style: %s",
+ str_compaction_style.c_str());
+
+ const auto& it_compaction_pri =
+ compaction_pri_to_string.find(compaction_pri);
+ std::string str_compaction_pri;
+ if (it_compaction_pri == compaction_pri_to_string.end()) {
+ assert(false);
+ str_compaction_pri = "unknown_" + std::to_string(compaction_pri);
+ } else {
+ str_compaction_pri = it_compaction_pri->second;
+ }
+ ROCKS_LOG_HEADER(log,
+ " Options.compaction_pri: %s",
+ str_compaction_pri.c_str());
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.size_ratio: %u",
+ compaction_options_universal.size_ratio);
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.min_merge_width: %u",
+ compaction_options_universal.min_merge_width);
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.max_merge_width: %u",
+ compaction_options_universal.max_merge_width);
+ ROCKS_LOG_HEADER(
+ log,
+ "Options.compaction_options_universal."
+ "max_size_amplification_percent: %u",
+ compaction_options_universal.max_size_amplification_percent);
+ ROCKS_LOG_HEADER(
+ log,
+ "Options.compaction_options_universal.compression_size_percent: %d",
+ compaction_options_universal.compression_size_percent);
+ const auto& it_compaction_stop_style = compaction_stop_style_to_string.find(
+ compaction_options_universal.stop_style);
+ std::string str_compaction_stop_style;
+ if (it_compaction_stop_style == compaction_stop_style_to_string.end()) {
+ assert(false);
+ str_compaction_stop_style =
+ "unknown_" + std::to_string(compaction_options_universal.stop_style);
+ } else {
+ str_compaction_stop_style = it_compaction_stop_style->second;
+ }
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_universal.stop_style: %s",
+ str_compaction_stop_style.c_str());
+ ROCKS_LOG_HEADER(
+ log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64,
+ compaction_options_fifo.max_table_files_size);
+ ROCKS_LOG_HEADER(log,
+ "Options.compaction_options_fifo.allow_compaction: %d",
+ compaction_options_fifo.allow_compaction);
+ std::ostringstream collector_info;
+ for (const auto& collector_factory : table_properties_collector_factories) {
+ collector_info << collector_factory->ToString() << ';';
+ }
+ ROCKS_LOG_HEADER(
+ log, " Options.table_properties_collectors: %s",
+ collector_info.str().c_str());
+ ROCKS_LOG_HEADER(log,
+ " Options.inplace_update_support: %d",
+ inplace_update_support);
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.inplace_update_num_locks: %" ROCKSDB_PRIszt,
+ inplace_update_num_locks);
+ // TODO: easier config for bloom (maybe based on avg key/value size)
+ ROCKS_LOG_HEADER(
+ log, " Options.memtable_prefix_bloom_size_ratio: %f",
+ memtable_prefix_bloom_size_ratio);
+ ROCKS_LOG_HEADER(log,
+ " Options.memtable_whole_key_filtering: %d",
+ memtable_whole_key_filtering);
+
+ ROCKS_LOG_HEADER(log, " Options.memtable_huge_page_size: %" ROCKSDB_PRIszt,
+ memtable_huge_page_size);
+ ROCKS_LOG_HEADER(log,
+ " Options.bloom_locality: %d",
+ bloom_locality);
+
+ ROCKS_LOG_HEADER(
+ log,
+ " Options.max_successive_merges: %" ROCKSDB_PRIszt,
+ max_successive_merges);
+ ROCKS_LOG_HEADER(log,
+ " Options.optimize_filters_for_hits: %d",
+ optimize_filters_for_hits);
+ ROCKS_LOG_HEADER(log, " Options.paranoid_file_checks: %d",
+ paranoid_file_checks);
+ ROCKS_LOG_HEADER(log, " Options.force_consistency_checks: %d",
+ force_consistency_checks);
+ ROCKS_LOG_HEADER(log, " Options.report_bg_io_stats: %d",
+ report_bg_io_stats);
+ ROCKS_LOG_HEADER(log, " Options.ttl: %" PRIu64,
+ ttl);
+ ROCKS_LOG_HEADER(log,
+ " Options.periodic_compaction_seconds: %" PRIu64,
+ periodic_compaction_seconds);
+ ROCKS_LOG_HEADER(log, " Options.preclude_last_level_data_seconds: %" PRIu64,
+ preclude_last_level_data_seconds);
+ ROCKS_LOG_HEADER(log, " Options.preserve_internal_time_seconds: %" PRIu64,
+ preserve_internal_time_seconds);
+ ROCKS_LOG_HEADER(log, " Options.enable_blob_files: %s",
+ enable_blob_files ? "true" : "false");
+ ROCKS_LOG_HEADER(
+ log, " Options.min_blob_size: %" PRIu64,
+ min_blob_size);
+ ROCKS_LOG_HEADER(
+ log, " Options.blob_file_size: %" PRIu64,
+ blob_file_size);
+ ROCKS_LOG_HEADER(log, " Options.blob_compression_type: %s",
+ CompressionTypeToString(blob_compression_type).c_str());
+ ROCKS_LOG_HEADER(log, " Options.enable_blob_garbage_collection: %s",
+ enable_blob_garbage_collection ? "true" : "false");
+ ROCKS_LOG_HEADER(log, " Options.blob_garbage_collection_age_cutoff: %f",
+ blob_garbage_collection_age_cutoff);
+ ROCKS_LOG_HEADER(log, "Options.blob_garbage_collection_force_threshold: %f",
+ blob_garbage_collection_force_threshold);
+ ROCKS_LOG_HEADER(
+ log, " Options.blob_compaction_readahead_size: %" PRIu64,
+ blob_compaction_readahead_size);
+ ROCKS_LOG_HEADER(log, " Options.blob_file_starting_level: %d",
+ blob_file_starting_level);
+ if (blob_cache) {
+ ROCKS_LOG_HEADER(log, " Options.blob_cache: %s",
+ blob_cache->Name());
+ ROCKS_LOG_HEADER(log, " blob_cache options: %s",
+ blob_cache->GetPrintableOptions().c_str());
+ ROCKS_LOG_HEADER(
+ log, " blob_cache prepopulated: %s",
+ prepopulate_blob_cache == PrepopulateBlobCache::kFlushOnly
+ ? "flush only"
+ : "disabled");
+ }
+ ROCKS_LOG_HEADER(log, "Options.experimental_mempurge_threshold: %f",
+ experimental_mempurge_threshold);
+} // ColumnFamilyOptions::Dump
+
+void Options::Dump(Logger* log) const {
+ DBOptions::Dump(log);
+ ColumnFamilyOptions::Dump(log);
+} // Options::Dump
+
+void Options::DumpCFOptions(Logger* log) const {
+ ColumnFamilyOptions::Dump(log);
+} // Options::DumpCFOptions
+
+//
+// The goal of this method is to create a configuration that
+// allows an application to write all files into L0 and
+// then do a single compaction to output all files into L1.
+Options*
+Options::PrepareForBulkLoad()
+{
+ // never slowdown ingest.
+ level0_file_num_compaction_trigger = (1<<30);
+ level0_slowdown_writes_trigger = (1<<30);
+ level0_stop_writes_trigger = (1<<30);
+ soft_pending_compaction_bytes_limit = 0;
+ hard_pending_compaction_bytes_limit = 0;
+
+ // no auto compactions please. The application should issue a
+ // manual compaction after all data is loaded into L0.
+ disable_auto_compactions = true;
+ // A manual compaction run should pick all files in L0 in
+ // a single compaction run.
+ max_compaction_bytes = (static_cast<uint64_t>(1) << 60);
+
+ // It is better to have only 2 levels, otherwise a manual
+ // compaction would compact at every possible level, thereby
+ // increasing the total time needed for compactions.
+ num_levels = 2;
+
+ // Need to allow more write buffers to allow more parallism
+ // of flushes.
+ max_write_buffer_number = 6;
+ min_write_buffer_number_to_merge = 1;
+
+ // When compaction is disabled, more parallel flush threads can
+ // help with write throughput.
+ max_background_flushes = 4;
+
+ // Prevent a memtable flush to automatically promote files
+ // to L1. This is helpful so that all files that are
+ // input to the manual compaction are all at L0.
+ max_background_compactions = 2;
+
+ // The compaction would create large files in L1.
+ target_file_size_base = 256 * 1024 * 1024;
+ return this;
+}
+
+Options* Options::OptimizeForSmallDb() {
+ // 16MB block cache
+ std::shared_ptr<Cache> cache = NewLRUCache(16 << 20);
+
+ ColumnFamilyOptions::OptimizeForSmallDb(&cache);
+ DBOptions::OptimizeForSmallDb(&cache);
+ return this;
+}
+
+Options* Options::DisableExtraChecks() {
+ // See https://github.com/facebook/rocksdb/issues/9354
+ force_consistency_checks = false;
+ // Considered but no clear performance impact seen:
+ // * check_flush_compaction_key_order
+ // * paranoid_checks
+ // * flush_verify_memtable_count
+ // By current API contract, not including
+ // * verify_checksums
+ // because checking storage data integrity is a more standard practice.
+ return this;
+}
+
+Options* Options::OldDefaults(int rocksdb_major_version,
+ int rocksdb_minor_version) {
+ ColumnFamilyOptions::OldDefaults(rocksdb_major_version,
+ rocksdb_minor_version);
+ DBOptions::OldDefaults(rocksdb_major_version, rocksdb_minor_version);
+ return this;
+}
+
+DBOptions* DBOptions::OldDefaults(int rocksdb_major_version,
+ int rocksdb_minor_version) {
+ if (rocksdb_major_version < 4 ||
+ (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) {
+ max_file_opening_threads = 1;
+ table_cache_numshardbits = 4;
+ }
+ if (rocksdb_major_version < 5 ||
+ (rocksdb_major_version == 5 && rocksdb_minor_version < 2)) {
+ delayed_write_rate = 2 * 1024U * 1024U;
+ } else if (rocksdb_major_version < 5 ||
+ (rocksdb_major_version == 5 && rocksdb_minor_version < 6)) {
+ delayed_write_rate = 16 * 1024U * 1024U;
+ }
+ max_open_files = 5000;
+ wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OldDefaults(
+ int rocksdb_major_version, int rocksdb_minor_version) {
+ if (rocksdb_major_version < 5 ||
+ (rocksdb_major_version == 5 && rocksdb_minor_version <= 18)) {
+ compaction_pri = CompactionPri::kByCompensatedSize;
+ }
+ if (rocksdb_major_version < 4 ||
+ (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) {
+ write_buffer_size = 4 << 20;
+ target_file_size_base = 2 * 1048576;
+ max_bytes_for_level_base = 10 * 1048576;
+ soft_pending_compaction_bytes_limit = 0;
+ hard_pending_compaction_bytes_limit = 0;
+ }
+ if (rocksdb_major_version < 5) {
+ level0_stop_writes_trigger = 24;
+ } else if (rocksdb_major_version == 5 && rocksdb_minor_version < 2) {
+ level0_stop_writes_trigger = 30;
+ }
+
+ return this;
+}
+
+// Optimization functions
+DBOptions* DBOptions::OptimizeForSmallDb(std::shared_ptr<Cache>* cache) {
+ max_file_opening_threads = 1;
+ max_open_files = 5000;
+
+ // Cost memtable to block cache too.
+ std::shared_ptr<ROCKSDB_NAMESPACE::WriteBufferManager> wbm =
+ std::make_shared<ROCKSDB_NAMESPACE::WriteBufferManager>(
+ 0, (cache != nullptr) ? *cache : std::shared_ptr<Cache>());
+ write_buffer_manager = wbm;
+
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForSmallDb(
+ std::shared_ptr<Cache>* cache) {
+ write_buffer_size = 2 << 20;
+ target_file_size_base = 2 * 1048576;
+ max_bytes_for_level_base = 10 * 1048576;
+ soft_pending_compaction_bytes_limit = 256 * 1048576;
+ hard_pending_compaction_bytes_limit = 1073741824ul;
+
+ BlockBasedTableOptions table_options;
+ table_options.block_cache =
+ (cache != nullptr) ? *cache : std::shared_ptr<Cache>();
+ table_options.cache_index_and_filter_blocks = true;
+ // Two level iterator to avoid LRU cache imbalance
+ table_options.index_type =
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+ table_factory.reset(new BlockBasedTableFactory(table_options));
+
+ return this;
+}
+
+#ifndef ROCKSDB_LITE
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(
+ uint64_t block_cache_size_mb) {
+ BlockBasedTableOptions block_based_options;
+ block_based_options.data_block_index_type =
+ BlockBasedTableOptions::kDataBlockBinaryAndHash;
+ block_based_options.data_block_hash_table_util_ratio = 0.75;
+ block_based_options.filter_policy.reset(NewBloomFilterPolicy(10));
+ block_based_options.block_cache =
+ NewLRUCache(static_cast<size_t>(block_cache_size_mb * 1024 * 1024));
+ table_factory.reset(new BlockBasedTableFactory(block_based_options));
+ memtable_prefix_bloom_size_ratio = 0.02;
+ memtable_whole_key_filtering = true;
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction(
+ uint64_t memtable_memory_budget) {
+ write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
+ // merge two memtables when flushing to L0
+ min_write_buffer_number_to_merge = 2;
+ // this means we'll use 50% extra memory in the worst case, but will reduce
+ // write stalls.
+ max_write_buffer_number = 6;
+ // start flushing L0->L1 as soon as possible. each file on level0 is
+ // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than
+ // memtable_memory_budget.
+ level0_file_num_compaction_trigger = 2;
+ // doesn't really matter much, but we don't want to create too many files
+ target_file_size_base = memtable_memory_budget / 8;
+ // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast
+ max_bytes_for_level_base = memtable_memory_budget;
+
+ // level style compaction
+ compaction_style = kCompactionStyleLevel;
+
+ // only compress levels >= 2
+ compression_per_level.resize(num_levels);
+ for (int i = 0; i < num_levels; ++i) {
+ if (i < 2) {
+ compression_per_level[i] = kNoCompression;
+ } else {
+ compression_per_level[i] =
+ LZ4_Supported()
+ ? kLZ4Compression
+ : (Snappy_Supported() ? kSnappyCompression : kNoCompression);
+ }
+ }
+ return this;
+}
+
+ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction(
+ uint64_t memtable_memory_budget) {
+ write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4);
+ // merge two memtables when flushing to L0
+ min_write_buffer_number_to_merge = 2;
+ // this means we'll use 50% extra memory in the worst case, but will reduce
+ // write stalls.
+ max_write_buffer_number = 6;
+ // universal style compaction
+ compaction_style = kCompactionStyleUniversal;
+ compaction_options_universal.compression_size_percent = 80;
+ return this;
+}
+
+DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
+ max_background_jobs = total_threads;
+ env->SetBackgroundThreads(total_threads, Env::LOW);
+ env->SetBackgroundThreads(1, Env::HIGH);
+ return this;
+}
+
+#endif // !ROCKSDB_LITE
+
+ReadOptions::ReadOptions()
+ : snapshot(nullptr),
+ iterate_lower_bound(nullptr),
+ iterate_upper_bound(nullptr),
+ readahead_size(0),
+ max_skippable_internal_keys(0),
+ read_tier(kReadAllTier),
+ verify_checksums(true),
+ fill_cache(true),
+ tailing(false),
+ managed(false),
+ total_order_seek(false),
+ auto_prefix_mode(false),
+ prefix_same_as_start(false),
+ pin_data(false),
+ background_purge_on_iterator_cleanup(false),
+ ignore_range_deletions(false),
+ timestamp(nullptr),
+ iter_start_ts(nullptr),
+ deadline(std::chrono::microseconds::zero()),
+ io_timeout(std::chrono::microseconds::zero()),
+ value_size_soft_limit(std::numeric_limits<uint64_t>::max()),
+ adaptive_readahead(false),
+ async_io(false),
+ optimize_multiget_for_io(true) {}
+
+ReadOptions::ReadOptions(bool cksum, bool cache)
+ : snapshot(nullptr),
+ iterate_lower_bound(nullptr),
+ iterate_upper_bound(nullptr),
+ readahead_size(0),
+ max_skippable_internal_keys(0),
+ read_tier(kReadAllTier),
+ verify_checksums(cksum),
+ fill_cache(cache),
+ tailing(false),
+ managed(false),
+ total_order_seek(false),
+ auto_prefix_mode(false),
+ prefix_same_as_start(false),
+ pin_data(false),
+ background_purge_on_iterator_cleanup(false),
+ ignore_range_deletions(false),
+ timestamp(nullptr),
+ iter_start_ts(nullptr),
+ deadline(std::chrono::microseconds::zero()),
+ io_timeout(std::chrono::microseconds::zero()),
+ value_size_soft_limit(std::numeric_limits<uint64_t>::max()),
+ adaptive_readahead(false),
+ async_io(false),
+ optimize_multiget_for_io(true) {}
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_helper.cc b/src/rocksdb/options/options_helper.cc
new file mode 100644
index 000000000..59b01e6fb
--- /dev/null
+++ b/src/rocksdb/options/options_helper.cc
@@ -0,0 +1,1478 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#include "options/options_helper.h"
+
+#include <cassert>
+#include <cctype>
+#include <cstdlib>
+#include <set>
+#include <unordered_set>
+#include <vector>
+
+#include "options/cf_options.h"
+#include "options/db_options.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/flush_block_policy.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/rate_limiter.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/table.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_type.h"
+#include "util/string_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+ConfigOptions::ConfigOptions()
+#ifndef ROCKSDB_LITE
+ : registry(ObjectRegistry::NewInstance())
+#endif
+{
+ env = Env::Default();
+}
+
+ConfigOptions::ConfigOptions(const DBOptions& db_opts) : env(db_opts.env) {
+#ifndef ROCKSDB_LITE
+ registry = ObjectRegistry::NewInstance();
+#endif
+}
+
+Status ValidateOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts) {
+ Status s;
+#ifndef ROCKSDB_LITE
+ auto db_cfg = DBOptionsAsConfigurable(db_opts);
+ auto cf_cfg = CFOptionsAsConfigurable(cf_opts);
+ s = db_cfg->ValidateOptions(db_opts, cf_opts);
+ if (s.ok()) s = cf_cfg->ValidateOptions(db_opts, cf_opts);
+#else
+ s = cf_opts.table_factory->ValidateOptions(db_opts, cf_opts);
+#endif
+ return s;
+}
+
+DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
+ const MutableDBOptions& mutable_db_options) {
+ DBOptions options;
+
+ options.create_if_missing = immutable_db_options.create_if_missing;
+ options.create_missing_column_families =
+ immutable_db_options.create_missing_column_families;
+ options.error_if_exists = immutable_db_options.error_if_exists;
+ options.paranoid_checks = immutable_db_options.paranoid_checks;
+ options.flush_verify_memtable_count =
+ immutable_db_options.flush_verify_memtable_count;
+ options.track_and_verify_wals_in_manifest =
+ immutable_db_options.track_and_verify_wals_in_manifest;
+ options.verify_sst_unique_id_in_manifest =
+ immutable_db_options.verify_sst_unique_id_in_manifest;
+ options.env = immutable_db_options.env;
+ options.rate_limiter = immutable_db_options.rate_limiter;
+ options.sst_file_manager = immutable_db_options.sst_file_manager;
+ options.info_log = immutable_db_options.info_log;
+ options.info_log_level = immutable_db_options.info_log_level;
+ options.max_open_files = mutable_db_options.max_open_files;
+ options.max_file_opening_threads =
+ immutable_db_options.max_file_opening_threads;
+ options.max_total_wal_size = mutable_db_options.max_total_wal_size;
+ options.statistics = immutable_db_options.statistics;
+ options.use_fsync = immutable_db_options.use_fsync;
+ options.db_paths = immutable_db_options.db_paths;
+ options.db_log_dir = immutable_db_options.db_log_dir;
+ options.wal_dir = immutable_db_options.wal_dir;
+ options.delete_obsolete_files_period_micros =
+ mutable_db_options.delete_obsolete_files_period_micros;
+ options.max_background_jobs = mutable_db_options.max_background_jobs;
+ options.max_background_compactions =
+ mutable_db_options.max_background_compactions;
+ options.bytes_per_sync = mutable_db_options.bytes_per_sync;
+ options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync;
+ options.strict_bytes_per_sync = mutable_db_options.strict_bytes_per_sync;
+ options.max_subcompactions = mutable_db_options.max_subcompactions;
+ options.max_background_flushes = mutable_db_options.max_background_flushes;
+ options.max_log_file_size = immutable_db_options.max_log_file_size;
+ options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll;
+ options.keep_log_file_num = immutable_db_options.keep_log_file_num;
+ options.recycle_log_file_num = immutable_db_options.recycle_log_file_num;
+ options.max_manifest_file_size = immutable_db_options.max_manifest_file_size;
+ options.table_cache_numshardbits =
+ immutable_db_options.table_cache_numshardbits;
+ options.WAL_ttl_seconds = immutable_db_options.WAL_ttl_seconds;
+ options.WAL_size_limit_MB = immutable_db_options.WAL_size_limit_MB;
+ options.manifest_preallocation_size =
+ immutable_db_options.manifest_preallocation_size;
+ options.allow_mmap_reads = immutable_db_options.allow_mmap_reads;
+ options.allow_mmap_writes = immutable_db_options.allow_mmap_writes;
+ options.use_direct_reads = immutable_db_options.use_direct_reads;
+ options.use_direct_io_for_flush_and_compaction =
+ immutable_db_options.use_direct_io_for_flush_and_compaction;
+ options.allow_fallocate = immutable_db_options.allow_fallocate;
+ options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec;
+ options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec;
+ options.stats_persist_period_sec =
+ mutable_db_options.stats_persist_period_sec;
+ options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk;
+ options.stats_history_buffer_size =
+ mutable_db_options.stats_history_buffer_size;
+ options.advise_random_on_open = immutable_db_options.advise_random_on_open;
+ options.db_write_buffer_size = immutable_db_options.db_write_buffer_size;
+ options.write_buffer_manager = immutable_db_options.write_buffer_manager;
+ options.access_hint_on_compaction_start =
+ immutable_db_options.access_hint_on_compaction_start;
+ options.compaction_readahead_size =
+ mutable_db_options.compaction_readahead_size;
+ options.random_access_max_buffer_size =
+ immutable_db_options.random_access_max_buffer_size;
+ options.writable_file_max_buffer_size =
+ mutable_db_options.writable_file_max_buffer_size;
+ options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex;
+ options.listeners = immutable_db_options.listeners;
+ options.enable_thread_tracking = immutable_db_options.enable_thread_tracking;
+ options.delayed_write_rate = mutable_db_options.delayed_write_rate;
+ options.enable_pipelined_write = immutable_db_options.enable_pipelined_write;
+ options.unordered_write = immutable_db_options.unordered_write;
+ options.allow_concurrent_memtable_write =
+ immutable_db_options.allow_concurrent_memtable_write;
+ options.enable_write_thread_adaptive_yield =
+ immutable_db_options.enable_write_thread_adaptive_yield;
+ options.max_write_batch_group_size_bytes =
+ immutable_db_options.max_write_batch_group_size_bytes;
+ options.write_thread_max_yield_usec =
+ immutable_db_options.write_thread_max_yield_usec;
+ options.write_thread_slow_yield_usec =
+ immutable_db_options.write_thread_slow_yield_usec;
+ options.skip_stats_update_on_db_open =
+ immutable_db_options.skip_stats_update_on_db_open;
+ options.skip_checking_sst_file_sizes_on_db_open =
+ immutable_db_options.skip_checking_sst_file_sizes_on_db_open;
+ options.wal_recovery_mode = immutable_db_options.wal_recovery_mode;
+ options.allow_2pc = immutable_db_options.allow_2pc;
+ options.row_cache = immutable_db_options.row_cache;
+#ifndef ROCKSDB_LITE
+ options.wal_filter = immutable_db_options.wal_filter;
+#endif // ROCKSDB_LITE
+ options.fail_if_options_file_error =
+ immutable_db_options.fail_if_options_file_error;
+ options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
+ options.avoid_flush_during_recovery =
+ immutable_db_options.avoid_flush_during_recovery;
+ options.avoid_flush_during_shutdown =
+ mutable_db_options.avoid_flush_during_shutdown;
+ options.allow_ingest_behind = immutable_db_options.allow_ingest_behind;
+ options.two_write_queues = immutable_db_options.two_write_queues;
+ options.manual_wal_flush = immutable_db_options.manual_wal_flush;
+ options.wal_compression = immutable_db_options.wal_compression;
+ options.atomic_flush = immutable_db_options.atomic_flush;
+ options.avoid_unnecessary_blocking_io =
+ immutable_db_options.avoid_unnecessary_blocking_io;
+ options.log_readahead_size = immutable_db_options.log_readahead_size;
+ options.file_checksum_gen_factory =
+ immutable_db_options.file_checksum_gen_factory;
+ options.best_efforts_recovery = immutable_db_options.best_efforts_recovery;
+ options.max_bgerror_resume_count =
+ immutable_db_options.max_bgerror_resume_count;
+ options.bgerror_resume_retry_interval =
+ immutable_db_options.bgerror_resume_retry_interval;
+ options.db_host_id = immutable_db_options.db_host_id;
+ options.allow_data_in_errors = immutable_db_options.allow_data_in_errors;
+ options.checksum_handoff_file_types =
+ immutable_db_options.checksum_handoff_file_types;
+ options.lowest_used_cache_tier = immutable_db_options.lowest_used_cache_tier;
+ options.enforce_single_del_contracts =
+ immutable_db_options.enforce_single_del_contracts;
+ return options;
+}
+
+ColumnFamilyOptions BuildColumnFamilyOptions(
+ const ColumnFamilyOptions& options,
+ const MutableCFOptions& mutable_cf_options) {
+ ColumnFamilyOptions cf_opts(options);
+ UpdateColumnFamilyOptions(mutable_cf_options, &cf_opts);
+ // TODO(yhchiang): find some way to handle the following derived options
+ // * max_file_size
+ return cf_opts;
+}
+
+void UpdateColumnFamilyOptions(const MutableCFOptions& moptions,
+ ColumnFamilyOptions* cf_opts) {
+ // Memtable related options
+ cf_opts->write_buffer_size = moptions.write_buffer_size;
+ cf_opts->max_write_buffer_number = moptions.max_write_buffer_number;
+ cf_opts->arena_block_size = moptions.arena_block_size;
+ cf_opts->memtable_prefix_bloom_size_ratio =
+ moptions.memtable_prefix_bloom_size_ratio;
+ cf_opts->memtable_whole_key_filtering = moptions.memtable_whole_key_filtering;
+ cf_opts->memtable_huge_page_size = moptions.memtable_huge_page_size;
+ cf_opts->max_successive_merges = moptions.max_successive_merges;
+ cf_opts->inplace_update_num_locks = moptions.inplace_update_num_locks;
+ cf_opts->prefix_extractor = moptions.prefix_extractor;
+ cf_opts->experimental_mempurge_threshold =
+ moptions.experimental_mempurge_threshold;
+ cf_opts->memtable_protection_bytes_per_key =
+ moptions.memtable_protection_bytes_per_key;
+
+ // Compaction related options
+ cf_opts->disable_auto_compactions = moptions.disable_auto_compactions;
+ cf_opts->soft_pending_compaction_bytes_limit =
+ moptions.soft_pending_compaction_bytes_limit;
+ cf_opts->hard_pending_compaction_bytes_limit =
+ moptions.hard_pending_compaction_bytes_limit;
+ cf_opts->level0_file_num_compaction_trigger =
+ moptions.level0_file_num_compaction_trigger;
+ cf_opts->level0_slowdown_writes_trigger =
+ moptions.level0_slowdown_writes_trigger;
+ cf_opts->level0_stop_writes_trigger = moptions.level0_stop_writes_trigger;
+ cf_opts->max_compaction_bytes = moptions.max_compaction_bytes;
+ cf_opts->ignore_max_compaction_bytes_for_input =
+ moptions.ignore_max_compaction_bytes_for_input;
+ cf_opts->target_file_size_base = moptions.target_file_size_base;
+ cf_opts->target_file_size_multiplier = moptions.target_file_size_multiplier;
+ cf_opts->max_bytes_for_level_base = moptions.max_bytes_for_level_base;
+ cf_opts->max_bytes_for_level_multiplier =
+ moptions.max_bytes_for_level_multiplier;
+ cf_opts->ttl = moptions.ttl;
+ cf_opts->periodic_compaction_seconds = moptions.periodic_compaction_seconds;
+
+ cf_opts->max_bytes_for_level_multiplier_additional.clear();
+ for (auto value : moptions.max_bytes_for_level_multiplier_additional) {
+ cf_opts->max_bytes_for_level_multiplier_additional.emplace_back(value);
+ }
+
+ cf_opts->compaction_options_fifo = moptions.compaction_options_fifo;
+ cf_opts->compaction_options_universal = moptions.compaction_options_universal;
+
+ // Blob file related options
+ cf_opts->enable_blob_files = moptions.enable_blob_files;
+ cf_opts->min_blob_size = moptions.min_blob_size;
+ cf_opts->blob_file_size = moptions.blob_file_size;
+ cf_opts->blob_compression_type = moptions.blob_compression_type;
+ cf_opts->enable_blob_garbage_collection =
+ moptions.enable_blob_garbage_collection;
+ cf_opts->blob_garbage_collection_age_cutoff =
+ moptions.blob_garbage_collection_age_cutoff;
+ cf_opts->blob_garbage_collection_force_threshold =
+ moptions.blob_garbage_collection_force_threshold;
+ cf_opts->blob_compaction_readahead_size =
+ moptions.blob_compaction_readahead_size;
+ cf_opts->blob_file_starting_level = moptions.blob_file_starting_level;
+ cf_opts->prepopulate_blob_cache = moptions.prepopulate_blob_cache;
+
+ // Misc options
+ cf_opts->max_sequential_skip_in_iterations =
+ moptions.max_sequential_skip_in_iterations;
+ cf_opts->check_flush_compaction_key_order =
+ moptions.check_flush_compaction_key_order;
+ cf_opts->paranoid_file_checks = moptions.paranoid_file_checks;
+ cf_opts->report_bg_io_stats = moptions.report_bg_io_stats;
+ cf_opts->compression = moptions.compression;
+ cf_opts->compression_opts = moptions.compression_opts;
+ cf_opts->bottommost_compression = moptions.bottommost_compression;
+ cf_opts->bottommost_compression_opts = moptions.bottommost_compression_opts;
+ cf_opts->sample_for_compression = moptions.sample_for_compression;
+ cf_opts->compression_per_level = moptions.compression_per_level;
+ cf_opts->last_level_temperature = moptions.last_level_temperature;
+ cf_opts->bottommost_temperature = moptions.last_level_temperature;
+}
+
+void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions,
+ ColumnFamilyOptions* cf_opts) {
+ cf_opts->compaction_style = ioptions.compaction_style;
+ cf_opts->compaction_pri = ioptions.compaction_pri;
+ cf_opts->comparator = ioptions.user_comparator;
+ cf_opts->merge_operator = ioptions.merge_operator;
+ cf_opts->compaction_filter = ioptions.compaction_filter;
+ cf_opts->compaction_filter_factory = ioptions.compaction_filter_factory;
+ cf_opts->min_write_buffer_number_to_merge =
+ ioptions.min_write_buffer_number_to_merge;
+ cf_opts->max_write_buffer_number_to_maintain =
+ ioptions.max_write_buffer_number_to_maintain;
+ cf_opts->max_write_buffer_size_to_maintain =
+ ioptions.max_write_buffer_size_to_maintain;
+ cf_opts->inplace_update_support = ioptions.inplace_update_support;
+ cf_opts->inplace_callback = ioptions.inplace_callback;
+ cf_opts->memtable_factory = ioptions.memtable_factory;
+ cf_opts->table_factory = ioptions.table_factory;
+ cf_opts->table_properties_collector_factories =
+ ioptions.table_properties_collector_factories;
+ cf_opts->bloom_locality = ioptions.bloom_locality;
+ cf_opts->level_compaction_dynamic_level_bytes =
+ ioptions.level_compaction_dynamic_level_bytes;
+ cf_opts->level_compaction_dynamic_file_size =
+ ioptions.level_compaction_dynamic_file_size;
+ cf_opts->num_levels = ioptions.num_levels;
+ cf_opts->optimize_filters_for_hits = ioptions.optimize_filters_for_hits;
+ cf_opts->force_consistency_checks = ioptions.force_consistency_checks;
+ cf_opts->memtable_insert_with_hint_prefix_extractor =
+ ioptions.memtable_insert_with_hint_prefix_extractor;
+ cf_opts->cf_paths = ioptions.cf_paths;
+ cf_opts->compaction_thread_limiter = ioptions.compaction_thread_limiter;
+ cf_opts->sst_partitioner_factory = ioptions.sst_partitioner_factory;
+ cf_opts->blob_cache = ioptions.blob_cache;
+ cf_opts->preclude_last_level_data_seconds =
+ ioptions.preclude_last_level_data_seconds;
+ cf_opts->preserve_internal_time_seconds =
+ ioptions.preserve_internal_time_seconds;
+
+ // TODO(yhchiang): find some way to handle the following derived options
+ // * max_file_size
+}
+
+std::map<CompactionStyle, std::string>
+ OptionsHelper::compaction_style_to_string = {
+ {kCompactionStyleLevel, "kCompactionStyleLevel"},
+ {kCompactionStyleUniversal, "kCompactionStyleUniversal"},
+ {kCompactionStyleFIFO, "kCompactionStyleFIFO"},
+ {kCompactionStyleNone, "kCompactionStyleNone"}};
+
+std::map<CompactionPri, std::string> OptionsHelper::compaction_pri_to_string = {
+ {kByCompensatedSize, "kByCompensatedSize"},
+ {kOldestLargestSeqFirst, "kOldestLargestSeqFirst"},
+ {kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"},
+ {kMinOverlappingRatio, "kMinOverlappingRatio"},
+ {kRoundRobin, "kRoundRobin"}};
+
+std::map<CompactionStopStyle, std::string>
+ OptionsHelper::compaction_stop_style_to_string = {
+ {kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"},
+ {kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}};
+
+std::map<Temperature, std::string> OptionsHelper::temperature_to_string = {
+ {Temperature::kUnknown, "kUnknown"},
+ {Temperature::kHot, "kHot"},
+ {Temperature::kWarm, "kWarm"},
+ {Temperature::kCold, "kCold"}};
+
+std::unordered_map<std::string, ChecksumType>
+ OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum},
+ {"kCRC32c", kCRC32c},
+ {"kxxHash", kxxHash},
+ {"kxxHash64", kxxHash64},
+ {"kXXH3", kXXH3}};
+
+std::unordered_map<std::string, CompressionType>
+ OptionsHelper::compression_type_string_map = {
+ {"kNoCompression", kNoCompression},
+ {"kSnappyCompression", kSnappyCompression},
+ {"kZlibCompression", kZlibCompression},
+ {"kBZip2Compression", kBZip2Compression},
+ {"kLZ4Compression", kLZ4Compression},
+ {"kLZ4HCCompression", kLZ4HCCompression},
+ {"kXpressCompression", kXpressCompression},
+ {"kZSTD", kZSTD},
+ {"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
+ {"kDisableCompressionOption", kDisableCompressionOption}};
+
+std::vector<CompressionType> GetSupportedCompressions() {
+ // std::set internally to deduplicate potential name aliases
+ std::set<CompressionType> supported_compressions;
+ for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) {
+ CompressionType t = comp_to_name.second;
+ if (t != kDisableCompressionOption && CompressionTypeSupported(t)) {
+ supported_compressions.insert(t);
+ }
+ }
+ return std::vector<CompressionType>(supported_compressions.begin(),
+ supported_compressions.end());
+}
+
+std::vector<CompressionType> GetSupportedDictCompressions() {
+ std::set<CompressionType> dict_compression_types;
+ for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) {
+ CompressionType t = comp_to_name.second;
+ if (t != kDisableCompressionOption && DictCompressionTypeSupported(t)) {
+ dict_compression_types.insert(t);
+ }
+ }
+ return std::vector<CompressionType>(dict_compression_types.begin(),
+ dict_compression_types.end());
+}
+
+std::vector<ChecksumType> GetSupportedChecksums() {
+ std::set<ChecksumType> checksum_types;
+ for (const auto& e : OptionsHelper::checksum_type_string_map) {
+ checksum_types.insert(e.second);
+ }
+ return std::vector<ChecksumType>(checksum_types.begin(),
+ checksum_types.end());
+}
+
+#ifndef ROCKSDB_LITE
+static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type,
+ const std::string& value) {
+ switch (opt_type) {
+ case OptionType::kBoolean:
+ *static_cast<bool*>(opt_address) = ParseBoolean("", value);
+ break;
+ case OptionType::kInt:
+ *static_cast<int*>(opt_address) = ParseInt(value);
+ break;
+ case OptionType::kInt32T:
+ *static_cast<int32_t*>(opt_address) = ParseInt32(value);
+ break;
+ case OptionType::kInt64T:
+ PutUnaligned(static_cast<int64_t*>(opt_address), ParseInt64(value));
+ break;
+ case OptionType::kUInt:
+ *static_cast<unsigned int*>(opt_address) = ParseUint32(value);
+ break;
+ case OptionType::kUInt8T:
+ *static_cast<uint8_t*>(opt_address) = ParseUint8(value);
+ break;
+ case OptionType::kUInt32T:
+ *static_cast<uint32_t*>(opt_address) = ParseUint32(value);
+ break;
+ case OptionType::kUInt64T:
+ PutUnaligned(static_cast<uint64_t*>(opt_address), ParseUint64(value));
+ break;
+ case OptionType::kSizeT:
+ PutUnaligned(static_cast<size_t*>(opt_address), ParseSizeT(value));
+ break;
+ case OptionType::kString:
+ *static_cast<std::string*>(opt_address) = value;
+ break;
+ case OptionType::kDouble:
+ *static_cast<double*>(opt_address) = ParseDouble(value);
+ break;
+ case OptionType::kCompactionStyle:
+ return ParseEnum<CompactionStyle>(
+ compaction_style_string_map, value,
+ static_cast<CompactionStyle*>(opt_address));
+ case OptionType::kCompactionPri:
+ return ParseEnum<CompactionPri>(compaction_pri_string_map, value,
+ static_cast<CompactionPri*>(opt_address));
+ case OptionType::kCompressionType:
+ return ParseEnum<CompressionType>(
+ compression_type_string_map, value,
+ static_cast<CompressionType*>(opt_address));
+ case OptionType::kChecksumType:
+ return ParseEnum<ChecksumType>(checksum_type_string_map, value,
+ static_cast<ChecksumType*>(opt_address));
+ case OptionType::kEncodingType:
+ return ParseEnum<EncodingType>(encoding_type_string_map, value,
+ static_cast<EncodingType*>(opt_address));
+ case OptionType::kCompactionStopStyle:
+ return ParseEnum<CompactionStopStyle>(
+ compaction_stop_style_string_map, value,
+ static_cast<CompactionStopStyle*>(opt_address));
+ case OptionType::kEncodedString: {
+ std::string* output_addr = static_cast<std::string*>(opt_address);
+ (Slice(value)).DecodeHex(output_addr);
+ break;
+ }
+ case OptionType::kTemperature: {
+ return ParseEnum<Temperature>(temperature_string_map, value,
+ static_cast<Temperature*>(opt_address));
+ }
+ default:
+ return false;
+ }
+ return true;
+}
+
+bool SerializeSingleOptionHelper(const void* opt_address,
+ const OptionType opt_type,
+ std::string* value) {
+ assert(value);
+ switch (opt_type) {
+ case OptionType::kBoolean:
+ *value = *(static_cast<const bool*>(opt_address)) ? "true" : "false";
+ break;
+ case OptionType::kInt:
+ *value = std::to_string(*(static_cast<const int*>(opt_address)));
+ break;
+ case OptionType::kInt32T:
+ *value = std::to_string(*(static_cast<const int32_t*>(opt_address)));
+ break;
+ case OptionType::kInt64T:
+ {
+ int64_t v;
+ GetUnaligned(static_cast<const int64_t*>(opt_address), &v);
+ *value = std::to_string(v);
+ }
+ break;
+ case OptionType::kUInt:
+ *value = std::to_string(*(static_cast<const unsigned int*>(opt_address)));
+ break;
+ case OptionType::kUInt8T:
+ *value = std::to_string(*(static_cast<const uint8_t*>(opt_address)));
+ break;
+ case OptionType::kUInt32T:
+ *value = std::to_string(*(static_cast<const uint32_t*>(opt_address)));
+ break;
+ case OptionType::kUInt64T:
+ {
+ uint64_t v;
+ GetUnaligned(static_cast<const uint64_t*>(opt_address), &v);
+ *value = std::to_string(v);
+ }
+ break;
+ case OptionType::kSizeT:
+ {
+ size_t v;
+ GetUnaligned(static_cast<const size_t*>(opt_address), &v);
+ *value = std::to_string(v);
+ }
+ break;
+ case OptionType::kDouble:
+ *value = std::to_string(*(static_cast<const double*>(opt_address)));
+ break;
+ case OptionType::kString:
+ *value =
+ EscapeOptionString(*(static_cast<const std::string*>(opt_address)));
+ break;
+ case OptionType::kCompactionStyle:
+ return SerializeEnum<CompactionStyle>(
+ compaction_style_string_map,
+ *(static_cast<const CompactionStyle*>(opt_address)), value);
+ case OptionType::kCompactionPri:
+ return SerializeEnum<CompactionPri>(
+ compaction_pri_string_map,
+ *(static_cast<const CompactionPri*>(opt_address)), value);
+ case OptionType::kCompressionType:
+ return SerializeEnum<CompressionType>(
+ compression_type_string_map,
+ *(static_cast<const CompressionType*>(opt_address)), value);
+ break;
+ case OptionType::kChecksumType:
+ return SerializeEnum<ChecksumType>(
+ checksum_type_string_map,
+ *static_cast<const ChecksumType*>(opt_address), value);
+ case OptionType::kEncodingType:
+ return SerializeEnum<EncodingType>(
+ encoding_type_string_map,
+ *static_cast<const EncodingType*>(opt_address), value);
+ case OptionType::kCompactionStopStyle:
+ return SerializeEnum<CompactionStopStyle>(
+ compaction_stop_style_string_map,
+ *static_cast<const CompactionStopStyle*>(opt_address), value);
+ case OptionType::kEncodedString: {
+ const auto* ptr = static_cast<const std::string*>(opt_address);
+ *value = (Slice(*ptr)).ToString(true);
+ break;
+ }
+ case OptionType::kTemperature: {
+ return SerializeEnum<Temperature>(
+ temperature_string_map, *static_cast<const Temperature*>(opt_address),
+ value);
+ }
+ default:
+ return false;
+ }
+ return true;
+}
+
+template <typename T>
+Status ConfigureFromMap(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, std::string>& opt_map,
+ const std::string& option_name, Configurable* config, T* new_opts) {
+ Status s = config->ConfigureFromMap(config_options, opt_map);
+ if (s.ok()) {
+ *new_opts = *(config->GetOptions<T>(option_name));
+ }
+ return s;
+}
+
+
+Status StringToMap(const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map) {
+ assert(opts_map);
+ // Example:
+ // opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
+ // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
+ size_t pos = 0;
+ std::string opts = trim(opts_str);
+ // If the input string starts and ends with "{...}", strip off the brackets
+ while (opts.size() > 2 && opts[0] == '{' && opts[opts.size() - 1] == '}') {
+ opts = trim(opts.substr(1, opts.size() - 2));
+ }
+
+ while (pos < opts.size()) {
+ size_t eq_pos = opts.find_first_of("={};", pos);
+ if (eq_pos == std::string::npos) {
+ return Status::InvalidArgument("Mismatched key value pair, '=' expected");
+ } else if (opts[eq_pos] != '=') {
+ return Status::InvalidArgument("Unexpected char in key");
+ }
+
+ std::string key = trim(opts.substr(pos, eq_pos - pos));
+ if (key.empty()) {
+ return Status::InvalidArgument("Empty key found");
+ }
+
+ std::string value;
+ Status s = OptionTypeInfo::NextToken(opts, ';', eq_pos + 1, &pos, &value);
+ if (!s.ok()) {
+ return s;
+ } else {
+ (*opts_map)[key] = value;
+ if (pos == std::string::npos) {
+ break;
+ } else {
+ pos++;
+ }
+ }
+ }
+
+ return Status::OK();
+}
+
+
+Status GetStringFromDBOptions(std::string* opt_string,
+ const DBOptions& db_options,
+ const std::string& delimiter) {
+ ConfigOptions config_options(db_options);
+ config_options.delimiter = delimiter;
+ return GetStringFromDBOptions(config_options, db_options, opt_string);
+}
+
+Status GetStringFromDBOptions(const ConfigOptions& config_options,
+ const DBOptions& db_options,
+ std::string* opt_string) {
+ assert(opt_string);
+ opt_string->clear();
+ auto config = DBOptionsAsConfigurable(db_options);
+ return config->GetOptionString(config_options, opt_string);
+}
+
+
+Status GetStringFromColumnFamilyOptions(std::string* opt_string,
+ const ColumnFamilyOptions& cf_options,
+ const std::string& delimiter) {
+ ConfigOptions config_options;
+ config_options.delimiter = delimiter;
+ return GetStringFromColumnFamilyOptions(config_options, cf_options,
+ opt_string);
+}
+
+Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options,
+ const ColumnFamilyOptions& cf_options,
+ std::string* opt_string) {
+ const auto config = CFOptionsAsConfigurable(cf_options);
+ return config->GetOptionString(config_options, opt_string);
+}
+
+Status GetStringFromCompressionType(std::string* compression_str,
+ CompressionType compression_type) {
+ bool ok = SerializeEnum<CompressionType>(compression_type_string_map,
+ compression_type, compression_str);
+ if (ok) {
+ return Status::OK();
+ } else {
+ return Status::InvalidArgument("Invalid compression types");
+ }
+}
+
+Status GetColumnFamilyOptionsFromMap(
+ const ColumnFamilyOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ ColumnFamilyOptions* new_options, bool input_strings_escaped,
+ bool ignore_unknown_options) {
+ ConfigOptions config_options;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ config_options.input_strings_escaped = input_strings_escaped;
+ return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
+}
+
+Status GetColumnFamilyOptionsFromMap(
+ const ConfigOptions& config_options,
+ const ColumnFamilyOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ ColumnFamilyOptions* new_options) {
+ assert(new_options);
+
+ *new_options = base_options;
+
+ const auto config = CFOptionsAsConfigurable(base_options);
+ Status s = ConfigureFromMap<ColumnFamilyOptions>(
+ config_options, opts_map, OptionsHelper::kCFOptionsName, config.get(),
+ new_options);
+ // Translate any errors (NotFound, NotSupported, to InvalidArgument
+ if (s.ok() || s.IsInvalidArgument()) {
+ return s;
+ } else {
+ return Status::InvalidArgument(s.getState());
+ }
+}
+
+Status GetColumnFamilyOptionsFromString(
+ const ColumnFamilyOptions& base_options,
+ const std::string& opts_str,
+ ColumnFamilyOptions* new_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+ return GetColumnFamilyOptionsFromString(config_options, base_options,
+ opts_str, new_options);
+}
+
+Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options,
+ const ColumnFamilyOptions& base_options,
+ const std::string& opts_str,
+ ColumnFamilyOptions* new_options) {
+ std::unordered_map<std::string, std::string> opts_map;
+ Status s = StringToMap(opts_str, &opts_map);
+ if (!s.ok()) {
+ *new_options = base_options;
+ return s;
+ }
+ return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
+}
+
+Status GetDBOptionsFromMap(
+ const DBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ DBOptions* new_options, bool input_strings_escaped,
+ bool ignore_unknown_options) {
+ ConfigOptions config_options(base_options);
+ config_options.input_strings_escaped = input_strings_escaped;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ return GetDBOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
+}
+
+Status GetDBOptionsFromMap(
+ const ConfigOptions& config_options, const DBOptions& base_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ DBOptions* new_options) {
+ assert(new_options);
+ *new_options = base_options;
+ auto config = DBOptionsAsConfigurable(base_options);
+ Status s = ConfigureFromMap<DBOptions>(config_options, opts_map,
+ OptionsHelper::kDBOptionsName,
+ config.get(), new_options);
+ // Translate any errors (NotFound, NotSupported, to InvalidArgument
+ if (s.ok() || s.IsInvalidArgument()) {
+ return s;
+ } else {
+ return Status::InvalidArgument(s.getState());
+ }
+}
+
+Status GetDBOptionsFromString(const DBOptions& base_options,
+ const std::string& opts_str,
+ DBOptions* new_options) {
+ ConfigOptions config_options(base_options);
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
+ return GetDBOptionsFromString(config_options, base_options, opts_str,
+ new_options);
+}
+
+Status GetDBOptionsFromString(const ConfigOptions& config_options,
+ const DBOptions& base_options,
+ const std::string& opts_str,
+ DBOptions* new_options) {
+ std::unordered_map<std::string, std::string> opts_map;
+ Status s = StringToMap(opts_str, &opts_map);
+ if (!s.ok()) {
+ *new_options = base_options;
+ return s;
+ }
+ return GetDBOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
+}
+
+Status GetOptionsFromString(const Options& base_options,
+ const std::string& opts_str, Options* new_options) {
+ ConfigOptions config_options(base_options);
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
+ return GetOptionsFromString(config_options, base_options, opts_str,
+ new_options);
+}
+
+Status GetOptionsFromString(const ConfigOptions& config_options,
+ const Options& base_options,
+ const std::string& opts_str, Options* new_options) {
+ ColumnFamilyOptions new_cf_options;
+ std::unordered_map<std::string, std::string> unused_opts;
+ std::unordered_map<std::string, std::string> opts_map;
+
+ assert(new_options);
+ *new_options = base_options;
+ Status s = StringToMap(opts_str, &opts_map);
+ if (!s.ok()) {
+ return s;
+ }
+ auto config = DBOptionsAsConfigurable(base_options);
+ s = config->ConfigureFromMap(config_options, opts_map, &unused_opts);
+
+ if (s.ok()) {
+ DBOptions* new_db_options =
+ config->GetOptions<DBOptions>(OptionsHelper::kDBOptionsName);
+ if (!unused_opts.empty()) {
+ s = GetColumnFamilyOptionsFromMap(config_options, base_options,
+ unused_opts, &new_cf_options);
+ if (s.ok()) {
+ *new_options = Options(*new_db_options, new_cf_options);
+ }
+ } else {
+ *new_options = Options(*new_db_options, base_options);
+ }
+ }
+ // Translate any errors (NotFound, NotSupported, to InvalidArgument
+ if (s.ok() || s.IsInvalidArgument()) {
+ return s;
+ } else {
+ return Status::InvalidArgument(s.getState());
+ }
+}
+
+std::unordered_map<std::string, EncodingType>
+ OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain},
+ {"kPrefix", kPrefix}};
+
+std::unordered_map<std::string, CompactionStyle>
+ OptionsHelper::compaction_style_string_map = {
+ {"kCompactionStyleLevel", kCompactionStyleLevel},
+ {"kCompactionStyleUniversal", kCompactionStyleUniversal},
+ {"kCompactionStyleFIFO", kCompactionStyleFIFO},
+ {"kCompactionStyleNone", kCompactionStyleNone}};
+
+std::unordered_map<std::string, CompactionPri>
+ OptionsHelper::compaction_pri_string_map = {
+ {"kByCompensatedSize", kByCompensatedSize},
+ {"kOldestLargestSeqFirst", kOldestLargestSeqFirst},
+ {"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst},
+ {"kMinOverlappingRatio", kMinOverlappingRatio},
+ {"kRoundRobin", kRoundRobin}};
+
+std::unordered_map<std::string, CompactionStopStyle>
+ OptionsHelper::compaction_stop_style_string_map = {
+ {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
+ {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
+
+std::unordered_map<std::string, Temperature>
+ OptionsHelper::temperature_string_map = {
+ {"kUnknown", Temperature::kUnknown},
+ {"kHot", Temperature::kHot},
+ {"kWarm", Temperature::kWarm},
+ {"kCold", Temperature::kCold}};
+
+std::unordered_map<std::string, PrepopulateBlobCache>
+ OptionsHelper::prepopulate_blob_cache_string_map = {
+ {"kDisable", PrepopulateBlobCache::kDisable},
+ {"kFlushOnly", PrepopulateBlobCache::kFlushOnly}};
+
+Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter,
+ size_t pos, size_t* end, std::string* token) {
+ while (pos < opts.size() && isspace(opts[pos])) {
+ ++pos;
+ }
+ // Empty value at the end
+ if (pos >= opts.size()) {
+ *token = "";
+ *end = std::string::npos;
+ return Status::OK();
+ } else if (opts[pos] == '{') {
+ int count = 1;
+ size_t brace_pos = pos + 1;
+ while (brace_pos < opts.size()) {
+ if (opts[brace_pos] == '{') {
+ ++count;
+ } else if (opts[brace_pos] == '}') {
+ --count;
+ if (count == 0) {
+ break;
+ }
+ }
+ ++brace_pos;
+ }
+ // found the matching closing brace
+ if (count == 0) {
+ *token = trim(opts.substr(pos + 1, brace_pos - pos - 1));
+ // skip all whitespace and move to the next delimiter
+ // brace_pos points to the next position after the matching '}'
+ pos = brace_pos + 1;
+ while (pos < opts.size() && isspace(opts[pos])) {
+ ++pos;
+ }
+ if (pos < opts.size() && opts[pos] != delimiter) {
+ return Status::InvalidArgument("Unexpected chars after nested options");
+ }
+ *end = pos;
+ } else {
+ return Status::InvalidArgument(
+ "Mismatched curly braces for nested options");
+ }
+ } else {
+ *end = opts.find(delimiter, pos);
+ if (*end == std::string::npos) {
+ // It either ends with a trailing semi-colon or the last key-value pair
+ *token = trim(opts.substr(pos));
+ } else {
+ *token = trim(opts.substr(pos, *end - pos));
+ }
+ }
+ return Status::OK();
+}
+
+Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
+ const std::string& opt_name,
+ const std::string& value, void* opt_ptr) const {
+ if (IsDeprecated()) {
+ return Status::OK();
+ }
+ try {
+ const std::string& opt_value = config_options.input_strings_escaped
+ ? UnescapeOptionString(value)
+ : value;
+
+ if (opt_ptr == nullptr) {
+ return Status::NotFound("Could not find option", opt_name);
+ } else if (parse_func_ != nullptr) {
+ ConfigOptions copy = config_options;
+ copy.invoke_prepare_options = false;
+ void* opt_addr = GetOffset(opt_ptr);
+ return parse_func_(copy, opt_name, opt_value, opt_addr);
+ } else if (ParseOptionHelper(GetOffset(opt_ptr), type_, opt_value)) {
+ return Status::OK();
+ } else if (IsConfigurable()) {
+ // The option is <config>.<name>
+ Configurable* config = AsRawPointer<Configurable>(opt_ptr);
+ if (opt_value.empty()) {
+ return Status::OK();
+ } else if (config == nullptr) {
+ return Status::NotFound("Could not find configurable: ", opt_name);
+ } else {
+ ConfigOptions copy = config_options;
+ copy.ignore_unknown_options = false;
+ copy.invoke_prepare_options = false;
+ if (opt_value.find("=") != std::string::npos) {
+ return config->ConfigureFromString(copy, opt_value);
+ } else {
+ return config->ConfigureOption(copy, opt_name, opt_value);
+ }
+ }
+ } else if (IsByName()) {
+ return Status::NotSupported("Deserializing the option " + opt_name +
+ " is not supported");
+ } else {
+ return Status::InvalidArgument("Error parsing:", opt_name);
+ }
+ } catch (std::exception& e) {
+ return Status::InvalidArgument("Error parsing " + opt_name + ":" +
+ std::string(e.what()));
+ }
+}
+
+Status OptionTypeInfo::ParseType(
+ const ConfigOptions& config_options, const std::string& opts_str,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_map,
+ void* opt_addr, std::unordered_map<std::string, std::string>* unused) {
+ std::unordered_map<std::string, std::string> opts_map;
+ Status status = StringToMap(opts_str, &opts_map);
+ if (!status.ok()) {
+ return status;
+ } else {
+ return ParseType(config_options, opts_map, type_map, opt_addr, unused);
+ }
+}
+
+Status OptionTypeInfo::ParseType(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, std::string>& opts_map,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_map,
+ void* opt_addr, std::unordered_map<std::string, std::string>* unused) {
+ for (const auto& opts_iter : opts_map) {
+ std::string opt_name;
+ const auto* opt_info = Find(opts_iter.first, type_map, &opt_name);
+ if (opt_info != nullptr) {
+ Status status =
+ opt_info->Parse(config_options, opt_name, opts_iter.second, opt_addr);
+ if (!status.ok()) {
+ return status;
+ }
+ } else if (unused != nullptr) {
+ (*unused)[opts_iter.first] = opts_iter.second;
+ } else if (!config_options.ignore_unknown_options) {
+ return Status::NotFound("Unrecognized option", opts_iter.first);
+ }
+ }
+ return Status::OK();
+}
+
+Status OptionTypeInfo::ParseStruct(
+ const ConfigOptions& config_options, const std::string& struct_name,
+ const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
+ const std::string& opt_name, const std::string& opt_value, void* opt_addr) {
+ assert(struct_map);
+ Status status;
+ if (opt_name == struct_name || EndsWith(opt_name, "." + struct_name)) {
+ // This option represents the entire struct
+ std::unordered_map<std::string, std::string> unused;
+ status =
+ ParseType(config_options, opt_value, *struct_map, opt_addr, &unused);
+ if (status.ok() && !unused.empty()) {
+ status = Status::InvalidArgument(
+ "Unrecognized option", struct_name + "." + unused.begin()->first);
+ }
+ } else if (StartsWith(opt_name, struct_name + ".")) {
+ // This option represents a nested field in the struct (e.g, struct.field)
+ std::string elem_name;
+ const auto opt_info =
+ Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
+ if (opt_info != nullptr) {
+ status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
+ } else {
+ status = Status::InvalidArgument("Unrecognized option", opt_name);
+ }
+ } else {
+ // This option represents a field in the struct (e.g. field)
+ std::string elem_name;
+ const auto opt_info = Find(opt_name, *struct_map, &elem_name);
+ if (opt_info != nullptr) {
+ status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
+ } else {
+ status = Status::InvalidArgument("Unrecognized option",
+ struct_name + "." + opt_name);
+ }
+ }
+ return status;
+}
+
+Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
+ const std::string& opt_name,
+ const void* const opt_ptr,
+ std::string* opt_value) const {
+ // If the option is no longer used in rocksdb and marked as deprecated,
+ // we skip it in the serialization.
+ if (opt_ptr == nullptr || IsDeprecated()) {
+ return Status::OK();
+ } else if (IsEnabled(OptionTypeFlags::kDontSerialize)) {
+ return Status::NotSupported("Cannot serialize option: ", opt_name);
+ } else if (serialize_func_ != nullptr) {
+ const void* opt_addr = GetOffset(opt_ptr);
+ return serialize_func_(config_options, opt_name, opt_addr, opt_value);
+ } else if (IsCustomizable()) {
+ const Customizable* custom = AsRawPointer<Customizable>(opt_ptr);
+ opt_value->clear();
+ if (custom == nullptr) {
+ // We do not have a custom object to serialize.
+ // If the option is not mutable and we are doing only mutable options,
+ // we return an empty string (which will cause the option not to be
+ // printed). Otherwise, we return the "nullptr" string, which will result
+ // in "option=nullptr" being printed.
+ if (IsMutable() || !config_options.mutable_options_only) {
+ *opt_value = kNullptrString;
+ } else {
+ *opt_value = "";
+ }
+ } else if (IsEnabled(OptionTypeFlags::kStringNameOnly) &&
+ !config_options.IsDetailed()) {
+ if (!config_options.mutable_options_only || IsMutable()) {
+ *opt_value = custom->GetId();
+ }
+ } else {
+ ConfigOptions embedded = config_options;
+ embedded.delimiter = ";";
+ // If this option is mutable, everything inside it should be considered
+ // mutable
+ if (IsMutable()) {
+ embedded.mutable_options_only = false;
+ }
+ std::string value = custom->ToString(embedded);
+ if (!embedded.mutable_options_only ||
+ value.find("=") != std::string::npos) {
+ *opt_value = value;
+ } else {
+ *opt_value = "";
+ }
+ }
+ return Status::OK();
+ } else if (IsConfigurable()) {
+ const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
+ if (config != nullptr) {
+ ConfigOptions embedded = config_options;
+ embedded.delimiter = ";";
+ *opt_value = config->ToString(embedded);
+ }
+ return Status::OK();
+ } else if (config_options.mutable_options_only && !IsMutable()) {
+ return Status::OK();
+ } else if (SerializeSingleOptionHelper(GetOffset(opt_ptr), type_,
+ opt_value)) {
+ return Status::OK();
+ } else {
+ return Status::InvalidArgument("Cannot serialize option: ", opt_name);
+ }
+}
+
+Status OptionTypeInfo::SerializeType(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_map,
+ const void* opt_addr, std::string* result) {
+ Status status;
+ for (const auto& iter : type_map) {
+ std::string single;
+ const auto& opt_info = iter.second;
+ if (opt_info.ShouldSerialize()) {
+ status =
+ opt_info.Serialize(config_options, iter.first, opt_addr, &single);
+ if (!status.ok()) {
+ return status;
+ } else {
+ result->append(iter.first + "=" + single + config_options.delimiter);
+ }
+ }
+ }
+ return status;
+}
+
+Status OptionTypeInfo::SerializeStruct(
+ const ConfigOptions& config_options, const std::string& struct_name,
+ const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
+ const std::string& opt_name, const void* opt_addr, std::string* value) {
+ assert(struct_map);
+ Status status;
+ if (EndsWith(opt_name, struct_name)) {
+ // We are going to write the struct as "{ prop1=value1; prop2=value2;}.
+ // Set the delimiter to ";" so that the everything will be on one line.
+ ConfigOptions embedded = config_options;
+ embedded.delimiter = ";";
+
+ // This option represents the entire struct
+ std::string result;
+ status = SerializeType(embedded, *struct_map, opt_addr, &result);
+ if (!status.ok()) {
+ return status;
+ } else {
+ *value = "{" + result + "}";
+ }
+ } else if (StartsWith(opt_name, struct_name + ".")) {
+ // This option represents a nested field in the struct (e.g, struct.field)
+ std::string elem_name;
+ const auto opt_info =
+ Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
+ if (opt_info != nullptr) {
+ status = opt_info->Serialize(config_options, elem_name, opt_addr, value);
+ } else {
+ status = Status::InvalidArgument("Unrecognized option", opt_name);
+ }
+ } else {
+ // This option represents a field in the struct (e.g. field)
+ std::string elem_name;
+ const auto opt_info = Find(opt_name, *struct_map, &elem_name);
+ if (opt_info == nullptr) {
+ status = Status::InvalidArgument("Unrecognized option", opt_name);
+ } else if (opt_info->ShouldSerialize()) {
+ status = opt_info->Serialize(config_options, opt_name + "." + elem_name,
+ opt_addr, value);
+ }
+ }
+ return status;
+}
+
+template <typename T>
+bool IsOptionEqual(const void* offset1, const void* offset2) {
+ return (*static_cast<const T*>(offset1) == *static_cast<const T*>(offset2));
+}
+
+static bool AreEqualDoubles(const double a, const double b) {
+ return (fabs(a - b) < 0.00001);
+}
+
+static bool AreOptionsEqual(OptionType type, const void* this_offset,
+ const void* that_offset) {
+ switch (type) {
+ case OptionType::kBoolean:
+ return IsOptionEqual<bool>(this_offset, that_offset);
+ case OptionType::kInt:
+ return IsOptionEqual<int>(this_offset, that_offset);
+ case OptionType::kUInt:
+ return IsOptionEqual<unsigned int>(this_offset, that_offset);
+ case OptionType::kInt32T:
+ return IsOptionEqual<int32_t>(this_offset, that_offset);
+ case OptionType::kInt64T: {
+ int64_t v1, v2;
+ GetUnaligned(static_cast<const int64_t*>(this_offset), &v1);
+ GetUnaligned(static_cast<const int64_t*>(that_offset), &v2);
+ return (v1 == v2);
+ }
+ case OptionType::kUInt8T:
+ return IsOptionEqual<uint8_t>(this_offset, that_offset);
+ case OptionType::kUInt32T:
+ return IsOptionEqual<uint32_t>(this_offset, that_offset);
+ case OptionType::kUInt64T: {
+ uint64_t v1, v2;
+ GetUnaligned(static_cast<const uint64_t*>(this_offset), &v1);
+ GetUnaligned(static_cast<const uint64_t*>(that_offset), &v2);
+ return (v1 == v2);
+ }
+ case OptionType::kSizeT: {
+ size_t v1, v2;
+ GetUnaligned(static_cast<const size_t*>(this_offset), &v1);
+ GetUnaligned(static_cast<const size_t*>(that_offset), &v2);
+ return (v1 == v2);
+ }
+ case OptionType::kString:
+ return IsOptionEqual<std::string>(this_offset, that_offset);
+ case OptionType::kDouble:
+ return AreEqualDoubles(*static_cast<const double*>(this_offset),
+ *static_cast<const double*>(that_offset));
+ case OptionType::kCompactionStyle:
+ return IsOptionEqual<CompactionStyle>(this_offset, that_offset);
+ case OptionType::kCompactionStopStyle:
+ return IsOptionEqual<CompactionStopStyle>(this_offset, that_offset);
+ case OptionType::kCompactionPri:
+ return IsOptionEqual<CompactionPri>(this_offset, that_offset);
+ case OptionType::kCompressionType:
+ return IsOptionEqual<CompressionType>(this_offset, that_offset);
+ case OptionType::kChecksumType:
+ return IsOptionEqual<ChecksumType>(this_offset, that_offset);
+ case OptionType::kEncodingType:
+ return IsOptionEqual<EncodingType>(this_offset, that_offset);
+ case OptionType::kEncodedString:
+ return IsOptionEqual<std::string>(this_offset, that_offset);
+ case OptionType::kTemperature:
+ return IsOptionEqual<Temperature>(this_offset, that_offset);
+ default:
+ return false;
+ } // End switch
+}
+
+bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
+ const std::string& opt_name,
+ const void* const this_ptr,
+ const void* const that_ptr,
+ std::string* mismatch) const {
+ auto level = GetSanityLevel();
+ if (!config_options.IsCheckEnabled(level)) {
+ return true; // If the sanity level is not being checked, skip it
+ }
+ if (this_ptr == nullptr || that_ptr == nullptr) {
+ if (this_ptr == that_ptr) {
+ return true;
+ }
+ } else if (equals_func_ != nullptr) {
+ const void* this_addr = GetOffset(this_ptr);
+ const void* that_addr = GetOffset(that_ptr);
+ if (equals_func_(config_options, opt_name, this_addr, that_addr,
+ mismatch)) {
+ return true;
+ }
+ } else {
+ const void* this_addr = GetOffset(this_ptr);
+ const void* that_addr = GetOffset(that_ptr);
+ if (AreOptionsEqual(type_, this_addr, that_addr)) {
+ return true;
+ } else if (IsConfigurable()) {
+ const auto* this_config = AsRawPointer<Configurable>(this_ptr);
+ const auto* that_config = AsRawPointer<Configurable>(that_ptr);
+ if (this_config == that_config) {
+ return true;
+ } else if (this_config != nullptr && that_config != nullptr) {
+ std::string bad_name;
+ bool matches;
+ if (level < config_options.sanity_level) {
+ ConfigOptions copy = config_options;
+ copy.sanity_level = level;
+ matches = this_config->AreEquivalent(copy, that_config, &bad_name);
+ } else {
+ matches = this_config->AreEquivalent(config_options, that_config,
+ &bad_name);
+ }
+ if (!matches) {
+ *mismatch = opt_name + "." + bad_name;
+ }
+ return matches;
+ }
+ }
+ }
+ if (mismatch->empty()) {
+ *mismatch = opt_name;
+ }
+ return false;
+}
+
+bool OptionTypeInfo::TypesAreEqual(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_map,
+ const void* this_addr, const void* that_addr, std::string* mismatch) {
+ for (const auto& iter : type_map) {
+ const auto& opt_info = iter.second;
+ if (!opt_info.AreEqual(config_options, iter.first, this_addr, that_addr,
+ mismatch)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool OptionTypeInfo::StructsAreEqual(
+ const ConfigOptions& config_options, const std::string& struct_name,
+ const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
+ const std::string& opt_name, const void* this_addr, const void* that_addr,
+ std::string* mismatch) {
+ assert(struct_map);
+ bool matches = true;
+ std::string result;
+ if (EndsWith(opt_name, struct_name)) {
+ // This option represents the entire struct
+ matches = TypesAreEqual(config_options, *struct_map, this_addr, that_addr,
+ &result);
+ if (!matches) {
+ *mismatch = struct_name + "." + result;
+ return false;
+ }
+ } else if (StartsWith(opt_name, struct_name + ".")) {
+ // This option represents a nested field in the struct (e.g, struct.field)
+ std::string elem_name;
+ const auto opt_info =
+ Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
+ assert(opt_info);
+ if (opt_info == nullptr) {
+ *mismatch = opt_name;
+ matches = false;
+ } else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
+ that_addr, &result)) {
+ matches = false;
+ *mismatch = struct_name + "." + result;
+ }
+ } else {
+ // This option represents a field in the struct (e.g. field)
+ std::string elem_name;
+ const auto opt_info = Find(opt_name, *struct_map, &elem_name);
+ assert(opt_info);
+ if (opt_info == nullptr) {
+ *mismatch = struct_name + "." + opt_name;
+ matches = false;
+ } else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
+ that_addr, &result)) {
+ matches = false;
+ *mismatch = struct_name + "." + result;
+ }
+ }
+ return matches;
+}
+
+bool MatchesOptionsTypeFromMap(
+ const ConfigOptions& config_options,
+ const std::unordered_map<std::string, OptionTypeInfo>& type_map,
+ const void* const this_ptr, const void* const that_ptr,
+ std::string* mismatch) {
+ for (auto& pair : type_map) {
+ // We skip checking deprecated variables as they might
+ // contain random values since they might not be initialized
+ if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) {
+ if (!pair.second.AreEqual(config_options, pair.first, this_ptr, that_ptr,
+ mismatch) &&
+ !pair.second.AreEqualByName(config_options, pair.first, this_ptr,
+ that_ptr)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
+ const std::string& opt_name,
+ const void* const this_ptr,
+ const void* const that_ptr) const {
+ if (IsByName()) {
+ std::string that_value;
+ if (Serialize(config_options, opt_name, that_ptr, &that_value).ok()) {
+ return AreEqualByName(config_options, opt_name, this_ptr, that_value);
+ }
+ }
+ return false;
+}
+
+bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
+ const std::string& opt_name,
+ const void* const opt_ptr,
+ const std::string& that_value) const {
+ std::string this_value;
+ if (!IsByName()) {
+ return false;
+ } else if (!Serialize(config_options, opt_name, opt_ptr, &this_value).ok()) {
+ return false;
+ } else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) {
+ if (that_value == kNullptrString) {
+ return true;
+ }
+ } else if (IsEnabled(OptionVerificationType::kByNameAllowNull)) {
+ if (that_value == kNullptrString) {
+ return true;
+ }
+ }
+ return (this_value == that_value);
+}
+
+Status OptionTypeInfo::Prepare(const ConfigOptions& config_options,
+ const std::string& name, void* opt_ptr) const {
+ if (ShouldPrepare()) {
+ if (prepare_func_ != nullptr) {
+ void* opt_addr = GetOffset(opt_ptr);
+ return prepare_func_(config_options, name, opt_addr);
+ } else if (IsConfigurable()) {
+ Configurable* config = AsRawPointer<Configurable>(opt_ptr);
+ if (config != nullptr) {
+ return config->PrepareOptions(config_options);
+ } else if (!CanBeNull()) {
+ return Status::NotFound("Missing configurable object", name);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status OptionTypeInfo::Validate(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts,
+ const std::string& name,
+ const void* opt_ptr) const {
+ if (ShouldValidate()) {
+ if (validate_func_ != nullptr) {
+ const void* opt_addr = GetOffset(opt_ptr);
+ return validate_func_(db_opts, cf_opts, name, opt_addr);
+ } else if (IsConfigurable()) {
+ const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
+ if (config != nullptr) {
+ return config->ValidateOptions(db_opts, cf_opts);
+ } else if (!CanBeNull()) {
+ return Status::NotFound("Missing configurable object", name);
+ }
+ }
+ }
+ return Status::OK();
+}
+
+const OptionTypeInfo* OptionTypeInfo::Find(
+ const std::string& opt_name,
+ const std::unordered_map<std::string, OptionTypeInfo>& opt_map,
+ std::string* elem_name) {
+ const auto iter = opt_map.find(opt_name); // Look up the value in the map
+ if (iter != opt_map.end()) { // Found the option in the map
+ *elem_name = opt_name; // Return the name
+ return &(iter->second); // Return the contents of the iterator
+ } else {
+ auto idx = opt_name.find("."); // Look for a separator
+ if (idx > 0 && idx != std::string::npos) { // We found a separator
+ auto siter =
+ opt_map.find(opt_name.substr(0, idx)); // Look for the short name
+ if (siter != opt_map.end()) { // We found the short name
+ if (siter->second.IsStruct() || // If the object is a struct
+ siter->second.IsConfigurable()) { // or a Configurable
+ *elem_name = opt_name.substr(idx + 1); // Return the rest
+ return &(siter->second); // Return the contents of the iterator
+ }
+ }
+ }
+ }
+ return nullptr;
+}
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_helper.h b/src/rocksdb/options/options_helper.h
new file mode 100644
index 000000000..7c751fc25
--- /dev/null
+++ b/src/rocksdb/options/options_helper.h
@@ -0,0 +1,122 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <map>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "rocksdb/advanced_options.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table.h"
+
+namespace ROCKSDB_NAMESPACE {
+struct ColumnFamilyOptions;
+struct ConfigOptions;
+struct DBOptions;
+struct ImmutableCFOptions;
+struct ImmutableDBOptions;
+struct MutableDBOptions;
+struct MutableCFOptions;
+struct Options;
+
+std::vector<CompressionType> GetSupportedCompressions();
+
+std::vector<CompressionType> GetSupportedDictCompressions();
+
+std::vector<ChecksumType> GetSupportedChecksums();
+
+inline bool IsSupportedChecksumType(ChecksumType type) {
+ // Avoid annoying compiler warning-as-error (-Werror=type-limits)
+ auto min = kNoChecksum;
+ auto max = kXXH3;
+ return type >= min && type <= max;
+}
+
+// Checks that the combination of DBOptions and ColumnFamilyOptions are valid
+Status ValidateOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts);
+
+DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
+ const MutableDBOptions& mutable_db_options);
+
+ColumnFamilyOptions BuildColumnFamilyOptions(
+ const ColumnFamilyOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options);
+
+void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions,
+ ColumnFamilyOptions* cf_opts);
+void UpdateColumnFamilyOptions(const MutableCFOptions& moptions,
+ ColumnFamilyOptions* cf_opts);
+
+#ifndef ROCKSDB_LITE
+std::unique_ptr<Configurable> DBOptionsAsConfigurable(
+ const MutableDBOptions& opts);
+std::unique_ptr<Configurable> DBOptionsAsConfigurable(
+ const DBOptions& opts,
+ const std::unordered_map<std::string, std::string>* opt_map = nullptr);
+std::unique_ptr<Configurable> CFOptionsAsConfigurable(
+ const MutableCFOptions& opts);
+std::unique_ptr<Configurable> CFOptionsAsConfigurable(
+ const ColumnFamilyOptions& opts,
+ const std::unordered_map<std::string, std::string>* opt_map = nullptr);
+
+extern Status StringToMap(
+ const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map);
+#endif // !ROCKSDB_LITE
+
+struct OptionsHelper {
+ static const std::string kCFOptionsName /*= "ColumnFamilyOptions"*/;
+ static const std::string kDBOptionsName /*= "DBOptions" */;
+ static std::map<CompactionStyle, std::string> compaction_style_to_string;
+ static std::map<CompactionPri, std::string> compaction_pri_to_string;
+ static std::map<CompactionStopStyle, std::string>
+ compaction_stop_style_to_string;
+ static std::map<Temperature, std::string> temperature_to_string;
+ static std::unordered_map<std::string, ChecksumType> checksum_type_string_map;
+ static std::unordered_map<std::string, CompressionType>
+ compression_type_string_map;
+ static std::unordered_map<std::string, PrepopulateBlobCache>
+ prepopulate_blob_cache_string_map;
+#ifndef ROCKSDB_LITE
+ static std::unordered_map<std::string, CompactionStopStyle>
+ compaction_stop_style_string_map;
+ static std::unordered_map<std::string, EncodingType> encoding_type_string_map;
+ static std::unordered_map<std::string, CompactionStyle>
+ compaction_style_string_map;
+ static std::unordered_map<std::string, CompactionPri>
+ compaction_pri_string_map;
+ static std::unordered_map<std::string, Temperature> temperature_string_map;
+#endif // !ROCKSDB_LITE
+};
+
+// Some aliasing
+static auto& compaction_style_to_string =
+ OptionsHelper::compaction_style_to_string;
+static auto& compaction_pri_to_string = OptionsHelper::compaction_pri_to_string;
+static auto& compaction_stop_style_to_string =
+ OptionsHelper::compaction_stop_style_to_string;
+static auto& temperature_to_string = OptionsHelper::temperature_to_string;
+static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map;
+#ifndef ROCKSDB_LITE
+static auto& compaction_stop_style_string_map =
+ OptionsHelper::compaction_stop_style_string_map;
+static auto& compression_type_string_map =
+ OptionsHelper::compression_type_string_map;
+static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map;
+static auto& compaction_style_string_map =
+ OptionsHelper::compaction_style_string_map;
+static auto& compaction_pri_string_map =
+ OptionsHelper::compaction_pri_string_map;
+static auto& temperature_string_map = OptionsHelper::temperature_string_map;
+static auto& prepopulate_blob_cache_string_map =
+ OptionsHelper::prepopulate_blob_cache_string_map;
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_parser.cc b/src/rocksdb/options/options_parser.cc
new file mode 100644
index 000000000..562a7b214
--- /dev/null
+++ b/src/rocksdb/options/options_parser.cc
@@ -0,0 +1,727 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include "options/options_parser.h"
+
+#include <cmath>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "file/line_file_reader.h"
+#include "file/writable_file_writer.h"
+#include "options/cf_options.h"
+#include "options/db_options.h"
+#include "options/options_helper.h"
+#include "port/port.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/db.h"
+#include "rocksdb/utilities/options_type.h"
+#include "test_util/sync_point.h"
+#include "util/cast_util.h"
+#include "util/string_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+static const std::string option_file_header =
+ "# This is a RocksDB option file.\n"
+ "#\n"
+ "# For detailed file format spec, please refer to the example file\n"
+ "# in examples/rocksdb_option_file_example.ini\n"
+ "#\n"
+ "\n";
+
+Status PersistRocksDBOptions(const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs) {
+ ConfigOptions
+ config_options; // Use default for escaped(true) and check (exact)
+ config_options.delimiter = "\n ";
+ // Do not invoke PrepareOptions when we are doing validation.
+ config_options.invoke_prepare_options = false;
+ // If a readahead size was set in the input options, use it
+ if (db_opt.log_readahead_size > 0) {
+ config_options.file_readahead_size = db_opt.log_readahead_size;
+ }
+ return PersistRocksDBOptions(config_options, db_opt, cf_names, cf_opts,
+ file_name, fs);
+}
+
+Status PersistRocksDBOptions(const ConfigOptions& config_options_in,
+ const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs) {
+ ConfigOptions config_options = config_options_in;
+ config_options.delimiter = "\n "; // Override the default to nl
+
+ TEST_SYNC_POINT("PersistRocksDBOptions:start");
+ if (cf_names.size() != cf_opts.size()) {
+ return Status::InvalidArgument(
+ "cf_names.size() and cf_opts.size() must be the same");
+ }
+ std::unique_ptr<FSWritableFile> wf;
+
+ Status s =
+ fs->NewWritableFile(file_name, FileOptions(), &wf, nullptr);
+ if (!s.ok()) {
+ return s;
+ }
+ std::unique_ptr<WritableFileWriter> writable;
+ writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(),
+ nullptr /* statistics */));
+
+ std::string options_file_content;
+
+ s = writable->Append(
+ option_file_header + "[" + opt_section_titles[kOptionSectionVersion] +
+ "]\n"
+ " rocksdb_version=" +
+ std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR) +
+ "." + std::to_string(ROCKSDB_PATCH) + "\n");
+ if (s.ok()) {
+ s = writable->Append(
+ " options_file_version=" + std::to_string(ROCKSDB_OPTION_FILE_MAJOR) +
+ "." + std::to_string(ROCKSDB_OPTION_FILE_MINOR) + "\n");
+ }
+ if (s.ok()) {
+ s = writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] +
+ "]\n ");
+ }
+
+ if (s.ok()) {
+ s = GetStringFromDBOptions(config_options, db_opt, &options_file_content);
+ }
+ if (s.ok()) {
+ s = writable->Append(options_file_content + "\n");
+ }
+
+ for (size_t i = 0; s.ok() && i < cf_opts.size(); ++i) {
+ // CFOptions section
+ s = writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] +
+ " \"" + EscapeOptionString(cf_names[i]) + "\"]\n ");
+ if (s.ok()) {
+ s = GetStringFromColumnFamilyOptions(config_options, cf_opts[i],
+ &options_file_content);
+ }
+ if (s.ok()) {
+ s = writable->Append(options_file_content + "\n");
+ }
+ // TableOptions section
+ auto* tf = cf_opts[i].table_factory.get();
+ if (tf != nullptr) {
+ if (s.ok()) {
+ s = writable->Append(
+ "[" + opt_section_titles[kOptionSectionTableOptions] + tf->Name() +
+ " \"" + EscapeOptionString(cf_names[i]) + "\"]\n ");
+ }
+ if (s.ok()) {
+ options_file_content.clear();
+ s = tf->GetOptionString(config_options, &options_file_content);
+ }
+ if (s.ok()) {
+ s = writable->Append(options_file_content + "\n");
+ }
+ }
+ }
+ if (s.ok()) {
+ s = writable->Sync(true /* use_fsync */);
+ }
+ if (s.ok()) {
+ s = writable->Close();
+ }
+ if (s.ok()) {
+ return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ config_options, db_opt, cf_names, cf_opts, file_name, fs);
+ }
+ return s;
+}
+
+RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }
+
+void RocksDBOptionsParser::Reset() {
+ db_opt_ = DBOptions();
+ db_opt_map_.clear();
+ cf_names_.clear();
+ cf_opts_.clear();
+ cf_opt_maps_.clear();
+ has_version_section_ = false;
+ has_db_options_ = false;
+ has_default_cf_options_ = false;
+ for (int i = 0; i < 3; ++i) {
+ db_version[i] = 0;
+ opt_file_version[i] = 0;
+ }
+}
+
+bool RocksDBOptionsParser::IsSection(const std::string& line) {
+ if (line.size() < 2) {
+ return false;
+ }
+ if (line[0] != '[' || line[line.size() - 1] != ']') {
+ return false;
+ }
+ return true;
+}
+
+Status RocksDBOptionsParser::ParseSection(OptionSection* section,
+ std::string* title,
+ std::string* argument,
+ const std::string& line,
+ const int line_num) {
+ *section = kOptionSectionUnknown;
+ // A section is of the form [<SectionName> "<SectionArg>"], where
+ // "<SectionArg>" is optional.
+ size_t arg_start_pos = line.find("\"");
+ size_t arg_end_pos = line.rfind("\"");
+ // The following if-then check tries to identify whether the input
+ // section has the optional section argument.
+ if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) {
+ *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true);
+ *argument = UnescapeOptionString(
+ line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1));
+ } else {
+ *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true);
+ *argument = "";
+ }
+ for (int i = 0; i < kOptionSectionUnknown; ++i) {
+ if (title->find(opt_section_titles[i]) == 0) {
+ if (i == kOptionSectionVersion || i == kOptionSectionDBOptions ||
+ i == kOptionSectionCFOptions) {
+ if (title->size() == opt_section_titles[i].size()) {
+ // if true, then it indicats equal
+ *section = static_cast<OptionSection>(i);
+ return CheckSection(*section, *argument, line_num);
+ }
+ } else if (i == kOptionSectionTableOptions) {
+ // This type of sections has a sufffix at the end of the
+ // section title
+ if (title->size() > opt_section_titles[i].size()) {
+ *section = static_cast<OptionSection>(i);
+ return CheckSection(*section, *argument, line_num);
+ }
+ }
+ }
+ }
+ return Status::InvalidArgument(std::string("Unknown section ") + line);
+}
+
+Status RocksDBOptionsParser::InvalidArgument(const int line_num,
+ const std::string& message) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error] ",
+ message + " (at line " + std::to_string(line_num) + ")");
+}
+
+Status RocksDBOptionsParser::ParseStatement(std::string* name,
+ std::string* value,
+ const std::string& line,
+ const int line_num) {
+ size_t eq_pos = line.find("=");
+ if (eq_pos == std::string::npos) {
+ return InvalidArgument(line_num, "A valid statement must have a '='.");
+ }
+
+ *name = TrimAndRemoveComment(line.substr(0, eq_pos), true);
+ *value =
+ TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1));
+ if (name->empty()) {
+ return InvalidArgument(line_num,
+ "A valid statement must have a variable name.");
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
+ bool ignore_unknown_options,
+ size_t file_readahead_size) {
+ ConfigOptions
+ config_options; // Use default for escaped(true) and check (exact)
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ if (file_readahead_size > 0) {
+ config_options.file_readahead_size = file_readahead_size;
+ }
+ return Parse(config_options, file_name, fs);
+}
+
+Status RocksDBOptionsParser::Parse(const ConfigOptions& config_options_in,
+ const std::string& file_name,
+ FileSystem* fs) {
+ Reset();
+ ConfigOptions config_options = config_options_in;
+
+ std::unique_ptr<FSSequentialFile> seq_file;
+ Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file,
+ nullptr);
+ if (!s.ok()) {
+ return s;
+ }
+ LineFileReader lf_reader(std::move(seq_file), file_name,
+ config_options.file_readahead_size);
+
+ OptionSection section = kOptionSectionUnknown;
+ std::string title;
+ std::string argument;
+ std::unordered_map<std::string, std::string> opt_map;
+ std::string line;
+ // we only support single-lined statement.
+ while (lf_reader.ReadLine(&line, Env::IO_TOTAL /* rate_limiter_priority */)) {
+ int line_num = static_cast<int>(lf_reader.GetLineNumber());
+ line = TrimAndRemoveComment(line);
+ if (line.empty()) {
+ continue;
+ }
+ if (IsSection(line)) {
+ s = EndSection(config_options, section, title, argument, opt_map);
+ opt_map.clear();
+ if (!s.ok()) {
+ return s;
+ }
+
+ // If the option file is not generated by a higher minor version,
+ // there shouldn't be any unknown option.
+ if (config_options.ignore_unknown_options &&
+ section == kOptionSectionVersion) {
+ if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR &&
+ db_version[1] <= ROCKSDB_MINOR)) {
+ config_options.ignore_unknown_options = false;
+ }
+ }
+
+ s = ParseSection(&section, &title, &argument, line, line_num);
+ if (!s.ok()) {
+ return s;
+ }
+ } else {
+ std::string name;
+ std::string value;
+ s = ParseStatement(&name, &value, line, line_num);
+ if (!s.ok()) {
+ return s;
+ }
+ opt_map.insert({name, value});
+ }
+ }
+ s = lf_reader.GetStatus();
+ if (!s.ok()) {
+ return s;
+ }
+
+ s = EndSection(config_options, section, title, argument, opt_map);
+ opt_map.clear();
+ if (!s.ok()) {
+ return s;
+ }
+ return ValidityCheck();
+}
+
+Status RocksDBOptionsParser::CheckSection(const OptionSection section,
+ const std::string& section_arg,
+ const int line_num) {
+ if (section == kOptionSectionDBOptions) {
+ if (has_db_options_) {
+ return InvalidArgument(
+ line_num,
+ "More than one DBOption section found in the option config file");
+ }
+ has_db_options_ = true;
+ } else if (section == kOptionSectionCFOptions) {
+ bool is_default_cf = (section_arg == kDefaultColumnFamilyName);
+ if (cf_opts_.size() == 0 && !is_default_cf) {
+ return InvalidArgument(
+ line_num,
+ "Default column family must be the first CFOptions section "
+ "in the option config file");
+ } else if (cf_opts_.size() != 0 && is_default_cf) {
+ return InvalidArgument(
+ line_num,
+ "Default column family must be the first CFOptions section "
+ "in the optio/n config file");
+ } else if (GetCFOptions(section_arg) != nullptr) {
+ return InvalidArgument(
+ line_num,
+ "Two identical column families found in option config file");
+ }
+ has_default_cf_options_ |= is_default_cf;
+ } else if (section == kOptionSectionTableOptions) {
+ if (GetCFOptions(section_arg) == nullptr) {
+ return InvalidArgument(
+ line_num, std::string(
+ "Does not find a matched column family name in "
+ "TableOptions section. Column Family Name:") +
+ section_arg);
+ }
+ } else if (section == kOptionSectionVersion) {
+ if (has_version_section_) {
+ return InvalidArgument(
+ line_num,
+ "More than one Version section found in the option config file.");
+ }
+ has_version_section_ = true;
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
+ const std::string& ver_string,
+ const int max_count,
+ int* version) {
+ int version_index = 0;
+ int current_number = 0;
+ int current_digit_count = 0;
+ bool has_dot = false;
+ for (int i = 0; i < max_count; ++i) {
+ version[i] = 0;
+ }
+ constexpr int kBufferSize = 200;
+ char buffer[kBufferSize];
+ for (size_t i = 0; i < ver_string.size(); ++i) {
+ if (ver_string[i] == '.') {
+ if (version_index >= max_count - 1) {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s can only contains at most %d dots.",
+ ver_name.c_str(), max_count - 1);
+ return Status::InvalidArgument(buffer);
+ }
+ if (current_digit_count == 0) {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s must have at least one digit before each dot.",
+ ver_name.c_str());
+ return Status::InvalidArgument(buffer);
+ }
+ version[version_index++] = current_number;
+ current_number = 0;
+ current_digit_count = 0;
+ has_dot = true;
+ } else if (isdigit(ver_string[i])) {
+ current_number = current_number * 10 + (ver_string[i] - '0');
+ current_digit_count++;
+ } else {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s can only contains dots and numbers.",
+ ver_name.c_str());
+ return Status::InvalidArgument(buffer);
+ }
+ }
+ version[version_index] = current_number;
+ if (has_dot && current_digit_count == 0) {
+ snprintf(buffer, sizeof(buffer) - 1,
+ "A valid %s must have at least one digit after each dot.",
+ ver_name.c_str());
+ return Status::InvalidArgument(buffer);
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::EndSection(
+ const ConfigOptions& config_options, const OptionSection section,
+ const std::string& section_title, const std::string& section_arg,
+ const std::unordered_map<std::string, std::string>& opt_map) {
+ Status s;
+ if (section == kOptionSectionDBOptions) {
+ s = GetDBOptionsFromMap(config_options, DBOptions(), opt_map, &db_opt_);
+ if (!s.ok()) {
+ return s;
+ }
+ db_opt_map_ = opt_map;
+ } else if (section == kOptionSectionCFOptions) {
+ // This condition should be ensured earlier in ParseSection
+ // so we make an assertion here.
+ assert(GetCFOptions(section_arg) == nullptr);
+ cf_names_.emplace_back(section_arg);
+ cf_opts_.emplace_back();
+ s = GetColumnFamilyOptionsFromMap(config_options, ColumnFamilyOptions(),
+ opt_map, &cf_opts_.back());
+ if (!s.ok()) {
+ return s;
+ }
+ // keep the parsed string.
+ cf_opt_maps_.emplace_back(opt_map);
+ } else if (section == kOptionSectionTableOptions) {
+ assert(GetCFOptions(section_arg) != nullptr);
+ auto* cf_opt = GetCFOptionsImpl(section_arg);
+ if (cf_opt == nullptr) {
+ return Status::InvalidArgument(
+ "The specified column family must be defined before the "
+ "TableOptions section:",
+ section_arg);
+ }
+ // Ignore error as table factory deserialization is optional
+ cf_opt->table_factory.reset();
+ s = TableFactory::CreateFromString(
+ config_options,
+ section_title.substr(
+ opt_section_titles[kOptionSectionTableOptions].size()),
+ &(cf_opt->table_factory));
+ if (s.ok() && cf_opt->table_factory != nullptr) {
+ s = cf_opt->table_factory->ConfigureFromMap(config_options, opt_map);
+ // Translate any errors (NotFound, NotSupported, to InvalidArgument
+ if (s.ok() || s.IsInvalidArgument()) {
+ return s;
+ } else {
+ return Status::InvalidArgument(s.getState());
+ }
+ } else {
+ // Return OK for not supported table factories as TableFactory
+ // Deserialization is optional.
+ cf_opt->table_factory.reset();
+ return Status::OK();
+ }
+ } else if (section == kOptionSectionVersion) {
+ for (const auto& pair : opt_map) {
+ if (pair.first == "rocksdb_version") {
+ s = ParseVersionNumber(pair.first, pair.second, 3, db_version);
+ if (!s.ok()) {
+ return s;
+ }
+ } else if (pair.first == "options_file_version") {
+ s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version);
+ if (!s.ok()) {
+ return s;
+ }
+ if (opt_file_version[0] < 1) {
+ return Status::InvalidArgument(
+ "A valid options_file_version must be at least 1.");
+ }
+ }
+ }
+ }
+ return s;
+}
+
+Status RocksDBOptionsParser::ValidityCheck() {
+ if (!has_db_options_) {
+ return Status::Corruption(
+ "A RocksDB Option file must have a single DBOptions section");
+ }
+ if (!has_default_cf_options_) {
+ return Status::Corruption(
+ "A RocksDB Option file must have a single CFOptions:default section");
+ }
+
+ return Status::OK();
+}
+
+std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line,
+ bool trim_only) {
+ size_t start = 0;
+ size_t end = line.size();
+
+ // we only support "#" style comment
+ if (!trim_only) {
+ size_t search_pos = 0;
+ while (search_pos < line.size()) {
+ size_t comment_pos = line.find('#', search_pos);
+ if (comment_pos == std::string::npos) {
+ break;
+ }
+ if (comment_pos == 0 || line[comment_pos - 1] != '\\') {
+ end = comment_pos;
+ break;
+ }
+ search_pos = comment_pos + 1;
+ }
+ }
+
+ while (start < end && isspace(line[start]) != 0) {
+ ++start;
+ }
+
+ // start < end implies end > 0.
+ while (start < end && isspace(line[end - 1]) != 0) {
+ --end;
+ }
+
+ if (start < end) {
+ return line.substr(start, end - start);
+ }
+
+ return "";
+}
+
+Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ const ConfigOptions& config_options_in, const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs) {
+ RocksDBOptionsParser parser;
+ ConfigOptions config_options = config_options_in;
+ config_options.invoke_prepare_options =
+ false; // No need to do a prepare for verify
+ if (config_options.sanity_level < ConfigOptions::kSanityLevelExactMatch) {
+ // If we are not doing an exact comparison, we should ignore
+ // unsupported options, as they may cause the Parse to fail
+ // (if the ObjectRegistry is not initialized)
+ config_options.ignore_unsupported_options = true;
+ }
+ Status s = parser.Parse(config_options, file_name, fs);
+ if (!s.ok()) {
+ return s;
+ }
+
+ // Verify DBOptions
+ s = VerifyDBOptions(config_options, db_opt, *parser.db_opt(),
+ parser.db_opt_map());
+ if (!s.ok()) {
+ return s;
+ }
+
+ // Verify ColumnFamily Name
+ if (cf_names.size() != parser.cf_names()->size()) {
+ if (config_options.sanity_level >=
+ ConfigOptions::kSanityLevelLooselyCompatible) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionParser Error] The persisted options does not have "
+ "the same number of column family names as the db instance.");
+ } else if (cf_opts.size() > parser.cf_opts()->size()) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error]",
+ "The persisted options file has less number of column family "
+ "names than that of the specified one.");
+ }
+ }
+ for (size_t i = 0; i < cf_names.size(); ++i) {
+ if (cf_names[i] != parser.cf_names()->at(i)) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionParser Error] The persisted options and the db"
+ "instance does not have the same name for column family ",
+ std::to_string(i));
+ }
+ }
+
+ // Verify Column Family Options
+ if (cf_opts.size() != parser.cf_opts()->size()) {
+ if (config_options.sanity_level >=
+ ConfigOptions::kSanityLevelLooselyCompatible) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error]",
+ "The persisted options does not have the same number of "
+ "column families as the db instance.");
+ } else if (cf_opts.size() > parser.cf_opts()->size()) {
+ return Status::InvalidArgument(
+ "[RocksDBOptionsParser Error]",
+ "The persisted options file has less number of column families "
+ "than that of the specified number.");
+ }
+ }
+ for (size_t i = 0; i < cf_opts.size(); ++i) {
+ s = VerifyCFOptions(config_options, cf_opts[i], parser.cf_opts()->at(i),
+ &(parser.cf_opt_maps()->at(i)));
+ if (!s.ok()) {
+ return s;
+ }
+ s = VerifyTableFactory(config_options, cf_opts[i].table_factory.get(),
+ parser.cf_opts()->at(i).table_factory.get());
+ if (!s.ok()) {
+ return s;
+ }
+ }
+
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::VerifyDBOptions(
+ const ConfigOptions& config_options, const DBOptions& base_opt,
+ const DBOptions& file_opt,
+ const std::unordered_map<std::string, std::string>* opt_map) {
+ auto base_config = DBOptionsAsConfigurable(base_opt, opt_map);
+ auto file_config = DBOptionsAsConfigurable(file_opt, opt_map);
+ std::string mismatch;
+ if (!base_config->AreEquivalent(config_options, file_config.get(),
+ &mismatch)) {
+ const size_t kBufferSize = 2048;
+ char buffer[kBufferSize];
+ std::string base_value;
+ std::string file_value;
+ int offset = snprintf(buffer, sizeof(buffer),
+ "[RocksDBOptionsParser]: "
+ "failed the verification on DBOptions::%s -- ",
+ mismatch.c_str());
+ Status s = base_config->GetOption(config_options, mismatch, &base_value);
+ if (s.ok()) {
+ s = file_config->GetOption(config_options, mismatch, &file_value);
+ }
+ assert(offset >= 0);
+ assert(static_cast<size_t>(offset) < sizeof(buffer));
+ if (s.ok()) {
+ snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
+ "-- The specified one is %s while the persisted one is %s.\n",
+ base_value.c_str(), file_value.c_str());
+ } else {
+ snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
+ "-- Unable to re-serialize an option: %s.\n",
+ s.ToString().c_str());
+ }
+ return Status::InvalidArgument(Slice(buffer, strlen(buffer)));
+ }
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::VerifyCFOptions(
+ const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt,
+ const ColumnFamilyOptions& file_opt,
+ const std::unordered_map<std::string, std::string>* opt_map) {
+ auto base_config = CFOptionsAsConfigurable(base_opt, opt_map);
+ auto file_config = CFOptionsAsConfigurable(file_opt, opt_map);
+ std::string mismatch;
+ if (!base_config->AreEquivalent(config_options, file_config.get(),
+ &mismatch)) {
+ std::string base_value;
+ std::string file_value;
+ // The options do not match
+ const size_t kBufferSize = 2048;
+ char buffer[kBufferSize];
+ Status s = base_config->GetOption(config_options, mismatch, &base_value);
+ if (s.ok()) {
+ s = file_config->GetOption(config_options, mismatch, &file_value);
+ }
+ int offset = snprintf(buffer, sizeof(buffer),
+ "[RocksDBOptionsParser]: "
+ "failed the verification on ColumnFamilyOptions::%s",
+ mismatch.c_str());
+ assert(offset >= 0);
+ assert(static_cast<size_t>(offset) < sizeof(buffer));
+ if (s.ok()) {
+ snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
+ "--- The specified one is %s while the persisted one is %s.\n",
+ base_value.c_str(), file_value.c_str());
+ } else {
+ snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
+ "--- Unable to re-serialize an option: %s.\n",
+ s.ToString().c_str());
+ }
+ return Status::InvalidArgument(Slice(buffer, sizeof(buffer)));
+ } // For each option
+ return Status::OK();
+}
+
+Status RocksDBOptionsParser::VerifyTableFactory(
+ const ConfigOptions& config_options, const TableFactory* base_tf,
+ const TableFactory* file_tf) {
+ std::string mismatch;
+ if (base_tf && file_tf) {
+ if (config_options.sanity_level > ConfigOptions::kSanityLevelNone &&
+ std::string(base_tf->Name()) != std::string(file_tf->Name())) {
+ return Status::Corruption(
+ "[RocksDBOptionsParser]: "
+ "failed the verification on TableFactory->Name()");
+ } else if (!base_tf->AreEquivalent(config_options, file_tf, &mismatch)) {
+ return Status::Corruption(std::string("[RocksDBOptionsParser]:"
+ "failed the verification on ") +
+ base_tf->Name() + "::",
+ mismatch);
+ }
+ } else {
+ // TODO(yhchiang): further support sanity check here
+ }
+ return Status::OK();
+}
+} // namespace ROCKSDB_NAMESPACE
+
+#endif // !ROCKSDB_LITE
diff --git a/src/rocksdb/options/options_parser.h b/src/rocksdb/options/options_parser.h
new file mode 100644
index 000000000..20e3d772d
--- /dev/null
+++ b/src/rocksdb/options/options_parser.h
@@ -0,0 +1,151 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+#ifndef ROCKSDB_LITE
+struct ConfigOptions;
+class OptionTypeInfo;
+class TableFactory;
+
+#define ROCKSDB_OPTION_FILE_MAJOR 1
+#define ROCKSDB_OPTION_FILE_MINOR 1
+
+enum OptionSection : char {
+ kOptionSectionVersion = 0,
+ kOptionSectionDBOptions,
+ kOptionSectionCFOptions,
+ kOptionSectionTableOptions,
+ kOptionSectionUnknown
+};
+
+static const std::string opt_section_titles[] = {
+ "Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"};
+
+Status PersistRocksDBOptions(const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs);
+Status PersistRocksDBOptions(const ConfigOptions& config_options,
+ const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs);
+
+class RocksDBOptionsParser {
+ public:
+ explicit RocksDBOptionsParser();
+ ~RocksDBOptionsParser() {}
+ void Reset();
+
+ // `file_readahead_size` is used for readahead for the option file.
+ // If 0 is given, a default value will be used.
+ Status Parse(const std::string& file_name, FileSystem* fs,
+ bool ignore_unknown_options, size_t file_readahead_size);
+
+ Status Parse(const ConfigOptions& config_options,
+ const std::string& file_name, FileSystem* fs);
+
+ static std::string TrimAndRemoveComment(const std::string& line,
+ const bool trim_only = false);
+
+ const DBOptions* db_opt() const { return &db_opt_; }
+ const std::unordered_map<std::string, std::string>* db_opt_map() const {
+ return &db_opt_map_;
+ }
+ const std::vector<ColumnFamilyOptions>* cf_opts() const { return &cf_opts_; }
+ const std::vector<std::string>* cf_names() const { return &cf_names_; }
+ const std::vector<std::unordered_map<std::string, std::string>>* cf_opt_maps()
+ const {
+ return &cf_opt_maps_;
+ }
+
+ const ColumnFamilyOptions* GetCFOptions(const std::string& name) {
+ return GetCFOptionsImpl(name);
+ }
+ size_t NumColumnFamilies() { return cf_opts_.size(); }
+ static Status VerifyRocksDBOptionsFromFile(
+ const ConfigOptions& config_options, const DBOptions& db_opt,
+ const std::vector<std::string>& cf_names,
+ const std::vector<ColumnFamilyOptions>& cf_opts,
+ const std::string& file_name, FileSystem* fs);
+ static Status VerifyDBOptions(
+ const ConfigOptions& config_options, const DBOptions& base_opt,
+ const DBOptions& new_opt,
+ const std::unordered_map<std::string, std::string>* new_opt_map =
+ nullptr);
+
+ static Status VerifyCFOptions(
+ const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt,
+ const ColumnFamilyOptions& new_opt,
+ const std::unordered_map<std::string, std::string>* new_opt_map =
+ nullptr);
+
+ static Status VerifyTableFactory(const ConfigOptions& config_options,
+ const TableFactory* base_tf,
+ const TableFactory* file_tf);
+
+ static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
+
+ static Status ParseStatement(std::string* name, std::string* value,
+ const std::string& line, const int line_num);
+
+ protected:
+ bool IsSection(const std::string& line);
+ Status ParseSection(OptionSection* section, std::string* title,
+ std::string* argument, const std::string& line,
+ const int line_num);
+
+ Status CheckSection(const OptionSection section,
+ const std::string& section_arg, const int line_num);
+
+ Status EndSection(
+ const ConfigOptions& config_options, const OptionSection section,
+ const std::string& title, const std::string& section_arg,
+ const std::unordered_map<std::string, std::string>& opt_map);
+
+ Status ValidityCheck();
+
+ static Status InvalidArgument(const int line_num, const std::string& message);
+
+ Status ParseVersionNumber(const std::string& ver_name,
+ const std::string& ver_string, const int max_count,
+ int* version);
+
+ ColumnFamilyOptions* GetCFOptionsImpl(const std::string& name) {
+ assert(cf_names_.size() == cf_opts_.size());
+ for (size_t i = 0; i < cf_names_.size(); ++i) {
+ if (cf_names_[i] == name) {
+ return &cf_opts_[i];
+ }
+ }
+ return nullptr;
+ }
+
+ private:
+ DBOptions db_opt_;
+ std::unordered_map<std::string, std::string> db_opt_map_;
+ std::vector<std::string> cf_names_;
+ std::vector<ColumnFamilyOptions> cf_opts_;
+ std::vector<std::unordered_map<std::string, std::string>> cf_opt_maps_;
+ bool has_version_section_;
+ bool has_db_options_;
+ bool has_default_cf_options_;
+ int db_version[3];
+ int opt_file_version[3];
+};
+
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
diff --git a/src/rocksdb/options/options_settable_test.cc b/src/rocksdb/options/options_settable_test.cc
new file mode 100644
index 000000000..63e9721ca
--- /dev/null
+++ b/src/rocksdb/options/options_settable_test.cc
@@ -0,0 +1,621 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <cstring>
+
+#include "options/cf_options.h"
+#include "options/db_options.h"
+#include "options/options_helper.h"
+#include "rocksdb/convenience.h"
+#include "test_util/testharness.h"
+
+#ifndef GFLAGS
+bool FLAGS_enable_print = false;
+#else
+#include "util/gflags_compat.h"
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+DEFINE_bool(enable_print, false, "Print options generated to console.");
+#endif // GFLAGS
+
+namespace ROCKSDB_NAMESPACE {
+
+// Verify options are settable from options strings.
+// We take the approach that depends on compiler behavior that copy constructor
+// won't touch implicit padding bytes, so that the test is fragile.
+// As a result, we only run the tests to verify new fields in options are
+// settable through string on limited platforms as it depends on behavior of
+// compilers.
+#ifndef ROCKSDB_LITE
+#if defined OS_LINUX || defined OS_WIN
+#ifndef __clang__
+#ifndef ROCKSDB_UBSAN_RUN
+
+class OptionsSettableTest : public testing::Test {
+ public:
+ OptionsSettableTest() {}
+};
+
+const char kSpecialChar = 'z';
+using OffsetGap = std::vector<std::pair<size_t, size_t>>;
+
+void FillWithSpecialChar(char* start_ptr, size_t total_size,
+ const OffsetGap& excluded,
+ char special_char = kSpecialChar) {
+ size_t offset = 0;
+ // The excluded vector contains pairs of bytes, (first, second).
+ // The first bytes are all set to the special char (represented as 'c' below).
+ // The second bytes are simply skipped (padding bytes).
+ // ccccc[skipped]cccccccc[skiped]cccccccc[skipped]
+ for (auto& pair : excluded) {
+ std::memset(start_ptr + offset, special_char, pair.first - offset);
+ offset = pair.first + pair.second;
+ }
+ // The rest of the structure is filled with the special characters.
+ // ccccc[skipped]cccccccc[skiped]cccccccc[skipped]cccccccccccccccc
+ std::memset(start_ptr + offset, special_char, total_size - offset);
+}
+
+int NumUnsetBytes(char* start_ptr, size_t total_size,
+ const OffsetGap& excluded) {
+ int total_unset_bytes_base = 0;
+ size_t offset = 0;
+ for (auto& pair : excluded) {
+ // The first part of the structure contains memory spaces that can be
+ // set (pair.first), and memory spaces that cannot be set (pair.second).
+ // Therefore total_unset_bytes_base only agregates bytes set to kSpecialChar
+ // in the pair.first bytes, but skips the pair.second bytes (padding bytes).
+ for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) {
+ if (*ptr == kSpecialChar) {
+ total_unset_bytes_base++;
+ }
+ }
+ offset = pair.first + pair.second;
+ }
+ // Then total_unset_bytes_base aggregates the bytes
+ // set to kSpecialChar in the rest of the structure
+ for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) {
+ if (*ptr == kSpecialChar) {
+ total_unset_bytes_base++;
+ }
+ }
+ return total_unset_bytes_base;
+}
+
+// Return true iff two structs are the same except excluded fields.
+bool CompareBytes(char* start_ptr1, char* start_ptr2, size_t total_size,
+ const OffsetGap& excluded) {
+ size_t offset = 0;
+ for (auto& pair : excluded) {
+ for (; offset < pair.first; offset++) {
+ if (*(start_ptr1 + offset) != *(start_ptr2 + offset)) {
+ return false;
+ }
+ }
+ offset = pair.first + pair.second;
+ }
+ for (; offset < total_size; offset++) {
+ if (*(start_ptr1 + offset) != *(start_ptr2 + offset)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// If the test fails, likely a new option is added to BlockBasedTableOptions
+// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
+// test is not updated accordingly.
+// After adding an option, we need to make sure it is settable by
+// GetBlockBasedTableOptionsFromString() and add the option to the input string
+// passed to the GetBlockBasedTableOptionsFromString() in this test.
+// If it is a complicated type, you also need to add the field to
+// kBbtoExcluded, and maybe add customized verification for it.
+TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
+ // Items in the form of <offset, size>. Need to be in ascending order
+ // and not overlapping. Need to update if new option to be excluded is added
+ // (e.g, pointer-type)
+ const OffsetGap kBbtoExcluded = {
+ {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
+ sizeof(std::shared_ptr<FlushBlockPolicyFactory>)},
+ {offsetof(struct BlockBasedTableOptions, block_cache),
+ sizeof(std::shared_ptr<Cache>)},
+ {offsetof(struct BlockBasedTableOptions, persistent_cache),
+ sizeof(std::shared_ptr<PersistentCache>)},
+ {offsetof(struct BlockBasedTableOptions, block_cache_compressed),
+ sizeof(std::shared_ptr<Cache>)},
+ {offsetof(struct BlockBasedTableOptions, cache_usage_options),
+ sizeof(CacheUsageOptions)},
+ {offsetof(struct BlockBasedTableOptions, filter_policy),
+ sizeof(std::shared_ptr<const FilterPolicy>)},
+ };
+
+ // In this test, we catch a new option of BlockBasedTableOptions that is not
+ // settable through GetBlockBasedTableOptionsFromString().
+ // We count padding bytes of the option struct, and assert it to be the same
+ // as unset bytes of an option struct initialized by
+ // GetBlockBasedTableOptionsFromString().
+
+ char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
+
+ // Count padding bytes by setting all bytes in the memory to a special char,
+ // copy a well constructed struct to this memory and see how many special
+ // bytes left.
+ BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions();
+ FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded);
+ // It based on the behavior of compiler that padding bytes are not changed
+ // when copying the struct. It's prone to failure when compiler behavior
+ // changes. We verify there is unset bytes to detect the case.
+ *bbto = BlockBasedTableOptions();
+ int unset_bytes_base =
+ NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded);
+ ASSERT_GT(unset_bytes_base, 0);
+ bbto->~BlockBasedTableOptions();
+
+ // Construct the base option passed into
+ // GetBlockBasedTableOptionsFromString().
+ bbto = new (bbto_ptr) BlockBasedTableOptions();
+ FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoExcluded);
+ // This option is not setable:
+ bbto->use_delta_encoding = true;
+
+ char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
+ BlockBasedTableOptions* new_bbto =
+ new (new_bbto_ptr) BlockBasedTableOptions();
+ FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions),
+ kBbtoExcluded);
+
+ // Need to update the option string if a new option is added.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ *bbto,
+ "cache_index_and_filter_blocks=1;"
+ "cache_index_and_filter_blocks_with_high_priority=true;"
+ "metadata_cache_options={top_level_index_pinning=kFallback;"
+ "partition_pinning=kAll;"
+ "unpartitioned_pinning=kFlushedAndSimilar;};"
+ "pin_l0_filter_and_index_blocks_in_cache=1;"
+ "pin_top_level_index_and_filter=1;"
+ "index_type=kHashSearch;"
+ "data_block_index_type=kDataBlockBinaryAndHash;"
+ "index_shortening=kNoShortening;"
+ "data_block_hash_table_util_ratio=0.75;"
+ "checksum=kxxHash;no_block_cache=1;"
+ "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
+ "block_size_deviation=8;block_restart_interval=4; "
+ "metadata_block_size=1024;"
+ "partition_filters=false;"
+ "optimize_filters_for_memory=true;"
+ "index_block_restart_interval=4;"
+ "filter_policy=bloomfilter:4:true;whole_key_filtering=1;detect_filter_"
+ "construct_corruption=false;"
+ "format_version=1;"
+ "verify_compression=true;read_amp_bytes_per_bit=0;"
+ "enable_index_compression=false;"
+ "block_align=true;"
+ "max_auto_readahead_size=0;"
+ "prepopulate_block_cache=kDisable;"
+ "initial_auto_readahead_size=0;"
+ "num_file_reads_for_auto_readahead=0",
+ new_bbto));
+
+ ASSERT_EQ(unset_bytes_base,
+ NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions),
+ kBbtoExcluded));
+
+ ASSERT_TRUE(new_bbto->block_cache.get() != nullptr);
+ ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr);
+ ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr);
+
+ bbto->~BlockBasedTableOptions();
+ new_bbto->~BlockBasedTableOptions();
+
+ delete[] bbto_ptr;
+ delete[] new_bbto_ptr;
+}
+
+// If the test fails, likely a new option is added to DBOptions
+// but it cannot be set through GetDBOptionsFromString(), or the test is not
+// updated accordingly.
+// After adding an option, we need to make sure it is settable by
+// GetDBOptionsFromString() and add the option to the input string passed to
+// DBOptionsFromString()in this test.
+// If it is a complicated type, you also need to add the field to
+// kDBOptionsExcluded, and maybe add customized verification for it.
+TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
+ const OffsetGap kDBOptionsExcluded = {
+ {offsetof(struct DBOptions, env), sizeof(Env*)},
+ {offsetof(struct DBOptions, rate_limiter),
+ sizeof(std::shared_ptr<RateLimiter>)},
+ {offsetof(struct DBOptions, sst_file_manager),
+ sizeof(std::shared_ptr<SstFileManager>)},
+ {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)},
+ {offsetof(struct DBOptions, statistics),
+ sizeof(std::shared_ptr<Statistics>)},
+ {offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)},
+ {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)},
+ {offsetof(struct DBOptions, wal_dir), sizeof(std::string)},
+ {offsetof(struct DBOptions, write_buffer_manager),
+ sizeof(std::shared_ptr<WriteBufferManager>)},
+ {offsetof(struct DBOptions, listeners),
+ sizeof(std::vector<std::shared_ptr<EventListener>>)},
+ {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)},
+ {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)},
+ {offsetof(struct DBOptions, file_checksum_gen_factory),
+ sizeof(std::shared_ptr<FileChecksumGenFactory>)},
+ {offsetof(struct DBOptions, db_host_id), sizeof(std::string)},
+ {offsetof(struct DBOptions, checksum_handoff_file_types),
+ sizeof(FileTypeSet)},
+ {offsetof(struct DBOptions, compaction_service),
+ sizeof(std::shared_ptr<CompactionService>)},
+ };
+
+ char* options_ptr = new char[sizeof(DBOptions)];
+
+ // Count padding bytes by setting all bytes in the memory to a special char,
+ // copy a well constructed struct to this memory and see how many special
+ // bytes left.
+ DBOptions* options = new (options_ptr) DBOptions();
+ FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsExcluded);
+ // It based on the behavior of compiler that padding bytes are not changed
+ // when copying the struct. It's prone to failure when compiler behavior
+ // changes. We verify there is unset bytes to detect the case.
+ *options = DBOptions();
+ int unset_bytes_base =
+ NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsExcluded);
+ ASSERT_GT(unset_bytes_base, 0);
+ options->~DBOptions();
+
+ options = new (options_ptr) DBOptions();
+ FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsExcluded);
+
+ char* new_options_ptr = new char[sizeof(DBOptions)];
+ DBOptions* new_options = new (new_options_ptr) DBOptions();
+ FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsExcluded);
+
+ // Need to update the option string if a new option is added.
+ ASSERT_OK(
+ GetDBOptionsFromString(*options,
+ "wal_bytes_per_sync=4295048118;"
+ "delete_obsolete_files_period_micros=4294967758;"
+ "WAL_ttl_seconds=4295008036;"
+ "WAL_size_limit_MB=4295036161;"
+ "max_write_batch_group_size_bytes=1048576;"
+ "wal_dir=path/to/wal_dir;"
+ "db_write_buffer_size=2587;"
+ "max_subcompactions=64330;"
+ "table_cache_numshardbits=28;"
+ "max_open_files=72;"
+ "max_file_opening_threads=35;"
+ "max_background_jobs=8;"
+ "max_background_compactions=33;"
+ "use_fsync=true;"
+ "use_adaptive_mutex=false;"
+ "max_total_wal_size=4295005604;"
+ "compaction_readahead_size=0;"
+ "keep_log_file_num=4890;"
+ "skip_stats_update_on_db_open=false;"
+ "skip_checking_sst_file_sizes_on_db_open=false;"
+ "max_manifest_file_size=4295009941;"
+ "db_log_dir=path/to/db_log_dir;"
+ "writable_file_max_buffer_size=1048576;"
+ "paranoid_checks=true;"
+ "flush_verify_memtable_count=true;"
+ "track_and_verify_wals_in_manifest=true;"
+ "verify_sst_unique_id_in_manifest=true;"
+ "is_fd_close_on_exec=false;"
+ "bytes_per_sync=4295013613;"
+ "strict_bytes_per_sync=true;"
+ "enable_thread_tracking=false;"
+ "recycle_log_file_num=0;"
+ "create_missing_column_families=true;"
+ "log_file_time_to_roll=3097;"
+ "max_background_flushes=35;"
+ "create_if_missing=false;"
+ "error_if_exists=true;"
+ "delayed_write_rate=4294976214;"
+ "manifest_preallocation_size=1222;"
+ "allow_mmap_writes=false;"
+ "stats_dump_period_sec=70127;"
+ "stats_persist_period_sec=54321;"
+ "persist_stats_to_disk=true;"
+ "stats_history_buffer_size=14159;"
+ "allow_fallocate=true;"
+ "allow_mmap_reads=false;"
+ "use_direct_reads=false;"
+ "use_direct_io_for_flush_and_compaction=false;"
+ "max_log_file_size=4607;"
+ "random_access_max_buffer_size=1048576;"
+ "advise_random_on_open=true;"
+ "fail_if_options_file_error=false;"
+ "enable_pipelined_write=false;"
+ "unordered_write=false;"
+ "allow_concurrent_memtable_write=true;"
+ "wal_recovery_mode=kPointInTimeRecovery;"
+ "enable_write_thread_adaptive_yield=true;"
+ "write_thread_slow_yield_usec=5;"
+ "write_thread_max_yield_usec=1000;"
+ "access_hint_on_compaction_start=NONE;"
+ "info_log_level=DEBUG_LEVEL;"
+ "dump_malloc_stats=false;"
+ "allow_2pc=false;"
+ "avoid_flush_during_recovery=false;"
+ "avoid_flush_during_shutdown=false;"
+ "allow_ingest_behind=false;"
+ "concurrent_prepare=false;"
+ "two_write_queues=false;"
+ "manual_wal_flush=false;"
+ "wal_compression=kZSTD;"
+ "seq_per_batch=false;"
+ "atomic_flush=false;"
+ "avoid_unnecessary_blocking_io=false;"
+ "log_readahead_size=0;"
+ "write_dbid_to_manifest=false;"
+ "best_efforts_recovery=false;"
+ "max_bgerror_resume_count=2;"
+ "bgerror_resume_retry_interval=1000000;"
+ "db_host_id=hostname;"
+ "lowest_used_cache_tier=kNonVolatileBlockTier;"
+ "allow_data_in_errors=false;"
+ "enforce_single_del_contracts=false;",
+ new_options));
+
+ ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
+ kDBOptionsExcluded));
+
+ options->~DBOptions();
+ new_options->~DBOptions();
+
+ delete[] options_ptr;
+ delete[] new_options_ptr;
+}
+
+// If the test fails, likely a new option is added to ColumnFamilyOptions
+// but it cannot be set through GetColumnFamilyOptionsFromString(), or the
+// test is not updated accordingly.
+// After adding an option, we need to make sure it is settable by
+// GetColumnFamilyOptionsFromString() and add the option to the input
+// string passed to GetColumnFamilyOptionsFromString() in this test.
+// If it is a complicated type, you also need to add the field to
+// kColumnFamilyOptionsExcluded, and maybe add customized verification
+// for it.
+TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
+ // options in the excluded set need to appear in the same order as in
+ // ColumnFamilyOptions.
+ const OffsetGap kColumnFamilyOptionsExcluded = {
+ {offsetof(struct ColumnFamilyOptions, inplace_callback),
+ sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))},
+ {offsetof(struct ColumnFamilyOptions,
+ memtable_insert_with_hint_prefix_extractor),
+ sizeof(std::shared_ptr<const SliceTransform>)},
+ {offsetof(struct ColumnFamilyOptions, compression_per_level),
+ sizeof(std::vector<CompressionType>)},
+ {offsetof(struct ColumnFamilyOptions,
+ max_bytes_for_level_multiplier_additional),
+ sizeof(std::vector<int>)},
+ {offsetof(struct ColumnFamilyOptions, memtable_factory),
+ sizeof(std::shared_ptr<MemTableRepFactory>)},
+ {offsetof(struct ColumnFamilyOptions,
+ table_properties_collector_factories),
+ sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)},
+ {offsetof(struct ColumnFamilyOptions, preclude_last_level_data_seconds),
+ sizeof(uint64_t)},
+ {offsetof(struct ColumnFamilyOptions, preserve_internal_time_seconds),
+ sizeof(uint64_t)},
+ {offsetof(struct ColumnFamilyOptions, blob_cache),
+ sizeof(std::shared_ptr<Cache>)},
+ {offsetof(struct ColumnFamilyOptions, comparator), sizeof(Comparator*)},
+ {offsetof(struct ColumnFamilyOptions, merge_operator),
+ sizeof(std::shared_ptr<MergeOperator>)},
+ {offsetof(struct ColumnFamilyOptions, compaction_filter),
+ sizeof(const CompactionFilter*)},
+ {offsetof(struct ColumnFamilyOptions, compaction_filter_factory),
+ sizeof(std::shared_ptr<CompactionFilterFactory>)},
+ {offsetof(struct ColumnFamilyOptions, prefix_extractor),
+ sizeof(std::shared_ptr<const SliceTransform>)},
+ {offsetof(struct ColumnFamilyOptions, snap_refresh_nanos),
+ sizeof(uint64_t)},
+ {offsetof(struct ColumnFamilyOptions, table_factory),
+ sizeof(std::shared_ptr<TableFactory>)},
+ {offsetof(struct ColumnFamilyOptions, cf_paths),
+ sizeof(std::vector<DbPath>)},
+ {offsetof(struct ColumnFamilyOptions, compaction_thread_limiter),
+ sizeof(std::shared_ptr<ConcurrentTaskLimiter>)},
+ {offsetof(struct ColumnFamilyOptions, sst_partitioner_factory),
+ sizeof(std::shared_ptr<SstPartitionerFactory>)},
+ };
+
+ char* options_ptr = new char[sizeof(ColumnFamilyOptions)];
+
+ // Count padding bytes by setting all bytes in the memory to a special char,
+ // copy a well constructed struct to this memory and see how many special
+ // bytes left.
+ FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsExcluded);
+
+ // Invoke a user-defined constructor in the hope that it does not overwrite
+ // padding bytes. Note that previously we relied on the implicitly-defined
+ // copy-assignment operator (i.e., `*options = ColumnFamilyOptions();`) here,
+ // which did in fact modify padding bytes.
+ ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions();
+
+ int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsExcluded);
+ ASSERT_GT(unset_bytes_base, 0);
+ options->~ColumnFamilyOptions();
+
+ options = new (options_ptr) ColumnFamilyOptions();
+ FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsExcluded);
+
+ // Following options are not settable through
+ // GetColumnFamilyOptionsFromString():
+ options->compaction_options_universal = CompactionOptionsUniversal();
+ options->num_levels = 42; // Initialize options for MutableCF
+ options->compaction_filter = nullptr;
+ options->sst_partitioner_factory = nullptr;
+
+ char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
+ ColumnFamilyOptions* new_options =
+ new (new_options_ptr) ColumnFamilyOptions();
+ FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsExcluded);
+
+ // Need to update the option string if a new option is added.
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ *options,
+ "compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
+ "table_factory=PlainTable;"
+ "prefix_extractor=rocksdb.CappedPrefix.13;"
+ "comparator=leveldb.BytewiseComparator;"
+ "compression_per_level=kBZip2Compression:kBZip2Compression:"
+ "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
+ "kSnappyCompression;"
+ "max_bytes_for_level_base=986;"
+ "bloom_locality=8016;"
+ "target_file_size_base=4294976376;"
+ "memtable_huge_page_size=2557;"
+ "max_successive_merges=5497;"
+ "max_sequential_skip_in_iterations=4294971408;"
+ "arena_block_size=1893;"
+ "target_file_size_multiplier=35;"
+ "min_write_buffer_number_to_merge=9;"
+ "max_write_buffer_number=84;"
+ "write_buffer_size=1653;"
+ "max_compaction_bytes=64;"
+ "ignore_max_compaction_bytes_for_input=true;"
+ "max_bytes_for_level_multiplier=60;"
+ "memtable_factory=SkipListFactory;"
+ "compression=kNoCompression;"
+ "compression_opts=5:6:7:8:9:10:true:11:false;"
+ "bottommost_compression_opts=4:5:6:7:8:9:true:10:true;"
+ "bottommost_compression=kDisableCompressionOption;"
+ "level0_stop_writes_trigger=33;"
+ "num_levels=99;"
+ "level0_slowdown_writes_trigger=22;"
+ "level0_file_num_compaction_trigger=14;"
+ "compaction_filter=urxcqstuwnCompactionFilter;"
+ "soft_pending_compaction_bytes_limit=0;"
+ "max_write_buffer_number_to_maintain=84;"
+ "max_write_buffer_size_to_maintain=2147483648;"
+ "merge_operator=aabcxehazrMergeOperator;"
+ "memtable_prefix_bloom_size_ratio=0.4642;"
+ "memtable_whole_key_filtering=true;"
+ "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"
+ "check_flush_compaction_key_order=false;"
+ "paranoid_file_checks=true;"
+ "force_consistency_checks=true;"
+ "inplace_update_num_locks=7429;"
+ "experimental_mempurge_threshold=0.0001;"
+ "optimize_filters_for_hits=false;"
+ "level_compaction_dynamic_level_bytes=false;"
+ "level_compaction_dynamic_file_size=true;"
+ "inplace_update_support=false;"
+ "compaction_style=kCompactionStyleFIFO;"
+ "compaction_pri=kMinOverlappingRatio;"
+ "hard_pending_compaction_bytes_limit=0;"
+ "disable_auto_compactions=false;"
+ "report_bg_io_stats=true;"
+ "ttl=60;"
+ "periodic_compaction_seconds=3600;"
+ "sample_for_compression=0;"
+ "enable_blob_files=true;"
+ "min_blob_size=256;"
+ "blob_file_size=1000000;"
+ "blob_compression_type=kBZip2Compression;"
+ "enable_blob_garbage_collection=true;"
+ "blob_garbage_collection_age_cutoff=0.5;"
+ "blob_garbage_collection_force_threshold=0.75;"
+ "blob_compaction_readahead_size=262144;"
+ "blob_file_starting_level=1;"
+ "prepopulate_blob_cache=kDisable;"
+ "bottommost_temperature=kWarm;"
+ "last_level_temperature=kWarm;"
+ "preclude_last_level_data_seconds=86400;"
+ "preserve_internal_time_seconds=86400;"
+ "compaction_options_fifo={max_table_files_size=3;allow_"
+ "compaction=false;age_for_warm=1;};"
+ "blob_cache=1M;"
+ "memtable_protection_bytes_per_key=2;",
+ new_options));
+
+ ASSERT_NE(new_options->blob_cache.get(), nullptr);
+
+ ASSERT_EQ(unset_bytes_base,
+ NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsExcluded));
+
+ ColumnFamilyOptions rnd_filled_options = *new_options;
+
+ options->~ColumnFamilyOptions();
+ new_options->~ColumnFamilyOptions();
+
+ delete[] options_ptr;
+ delete[] new_options_ptr;
+
+ // Test copying to mutabable and immutable options and copy back the mutable
+ // part.
+ const OffsetGap kMutableCFOptionsExcluded = {
+ {offsetof(struct MutableCFOptions, prefix_extractor),
+ sizeof(std::shared_ptr<const SliceTransform>)},
+ {offsetof(struct MutableCFOptions,
+ max_bytes_for_level_multiplier_additional),
+ sizeof(std::vector<int>)},
+ {offsetof(struct MutableCFOptions, compression_per_level),
+ sizeof(std::vector<CompressionType>)},
+ {offsetof(struct MutableCFOptions, max_file_size),
+ sizeof(std::vector<uint64_t>)},
+ };
+
+ // For all memory used for options, pre-fill every char. Otherwise, the
+ // padding bytes might be different so that byte-wise comparison doesn't
+ // general equal results even if objects are equal.
+ const char kMySpecialChar = 'x';
+ char* mcfo1_ptr = new char[sizeof(MutableCFOptions)];
+ FillWithSpecialChar(mcfo1_ptr, sizeof(MutableCFOptions),
+ kMutableCFOptionsExcluded, kMySpecialChar);
+ char* mcfo2_ptr = new char[sizeof(MutableCFOptions)];
+ FillWithSpecialChar(mcfo2_ptr, sizeof(MutableCFOptions),
+ kMutableCFOptionsExcluded, kMySpecialChar);
+
+ // A clean column family options is constructed after filling the same special
+ // char as the initial one. So that the padding bytes are the same.
+ char* cfo_clean_ptr = new char[sizeof(ColumnFamilyOptions)];
+ FillWithSpecialChar(cfo_clean_ptr, sizeof(ColumnFamilyOptions),
+ kColumnFamilyOptionsExcluded);
+ rnd_filled_options.num_levels = 66;
+ ColumnFamilyOptions* cfo_clean = new (cfo_clean_ptr) ColumnFamilyOptions();
+
+ MutableCFOptions* mcfo1 =
+ new (mcfo1_ptr) MutableCFOptions(rnd_filled_options);
+ ColumnFamilyOptions cfo_back = BuildColumnFamilyOptions(*cfo_clean, *mcfo1);
+ MutableCFOptions* mcfo2 = new (mcfo2_ptr) MutableCFOptions(cfo_back);
+
+ ASSERT_TRUE(CompareBytes(mcfo1_ptr, mcfo2_ptr, sizeof(MutableCFOptions),
+ kMutableCFOptionsExcluded));
+
+ cfo_clean->~ColumnFamilyOptions();
+ mcfo1->~MutableCFOptions();
+ mcfo2->~MutableCFOptions();
+ delete[] mcfo1_ptr;
+ delete[] mcfo2_ptr;
+ delete[] cfo_clean_ptr;
+}
+#endif // !ROCKSDB_UBSAN_RUN
+#endif // !__clang__
+#endif // OS_LINUX || OS_WIN
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+ ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+ ::testing::InitGoogleTest(&argc, argv);
+#ifdef GFLAGS
+ ParseCommandLineFlags(&argc, &argv, true);
+#endif // GFLAGS
+ return RUN_ALL_TESTS();
+}
diff --git a/src/rocksdb/options/options_test.cc b/src/rocksdb/options/options_test.cc
new file mode 100644
index 000000000..37001379a
--- /dev/null
+++ b/src/rocksdb/options/options_test.cc
@@ -0,0 +1,5014 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <cctype>
+#include <cinttypes>
+#include <cstring>
+#include <unordered_map>
+
+#include "cache/lru_cache.h"
+#include "cache/sharded_cache.h"
+#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/file_checksum.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/utilities/leveldb_options.h"
+#include "rocksdb/utilities/object_registry.h"
+#include "rocksdb/utilities/options_type.h"
+#include "table/block_based/filter_policy_internal.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+#include "util/random.h"
+#include "util/stderr_logger.h"
+#include "util/string_util.h"
+#include "utilities/merge_operators/bytesxor.h"
+#include "utilities/merge_operators/sortlist.h"
+#include "utilities/merge_operators/string_append/stringappend.h"
+#include "utilities/merge_operators/string_append/stringappend2.h"
+
+#ifndef GFLAGS
+bool FLAGS_enable_print = false;
+#else
+#include "util/gflags_compat.h"
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+DEFINE_bool(enable_print, false, "Print options generated to console.");
+#endif // GFLAGS
+
+namespace ROCKSDB_NAMESPACE {
+
+class OptionsTest : public testing::Test {};
+
+class UnregisteredTableFactory : public TableFactory {
+ public:
+ UnregisteredTableFactory() {}
+ const char* Name() const override { return "Unregistered"; }
+ using TableFactory::NewTableReader;
+ Status NewTableReader(const ReadOptions&, const TableReaderOptions&,
+ std::unique_ptr<RandomAccessFileReader>&&, uint64_t,
+ std::unique_ptr<TableReader>*, bool) const override {
+ return Status::NotSupported();
+ }
+ TableBuilder* NewTableBuilder(const TableBuilderOptions&,
+ WritableFileWriter*) const override {
+ return nullptr;
+ }
+};
+
+#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
+TEST_F(OptionsTest, GetOptionsFromMapTest) {
+ std::unordered_map<std::string, std::string> cf_options_map = {
+ {"write_buffer_size", "1"},
+ {"max_write_buffer_number", "2"},
+ {"min_write_buffer_number_to_merge", "3"},
+ {"max_write_buffer_number_to_maintain", "99"},
+ {"max_write_buffer_size_to_maintain", "-99999"},
+ {"compression", "kSnappyCompression"},
+ {"compression_per_level",
+ "kNoCompression:"
+ "kSnappyCompression:"
+ "kZlibCompression:"
+ "kBZip2Compression:"
+ "kLZ4Compression:"
+ "kLZ4HCCompression:"
+ "kXpressCompression:"
+ "kZSTD:"
+ "kZSTDNotFinalCompression"},
+ {"bottommost_compression", "kLZ4Compression"},
+ {"bottommost_compression_opts", "5:6:7:8:10:true"},
+ {"compression_opts", "4:5:6:7:8:2:true:100:false"},
+ {"num_levels", "8"},
+ {"level0_file_num_compaction_trigger", "8"},
+ {"level0_slowdown_writes_trigger", "9"},
+ {"level0_stop_writes_trigger", "10"},
+ {"target_file_size_base", "12"},
+ {"target_file_size_multiplier", "13"},
+ {"max_bytes_for_level_base", "14"},
+ {"level_compaction_dynamic_level_bytes", "true"},
+ {"max_bytes_for_level_multiplier", "15.0"},
+ {"max_bytes_for_level_multiplier_additional", "16:17:18"},
+ {"max_compaction_bytes", "21"},
+ {"hard_pending_compaction_bytes_limit", "211"},
+ {"arena_block_size", "22"},
+ {"disable_auto_compactions", "true"},
+ {"compaction_style", "kCompactionStyleLevel"},
+ {"compaction_pri", "kOldestSmallestSeqFirst"},
+ {"verify_checksums_in_compaction", "false"},
+ {"compaction_options_fifo", "23"},
+ {"max_sequential_skip_in_iterations", "24"},
+ {"inplace_update_support", "true"},
+ {"report_bg_io_stats", "true"},
+ {"compaction_measure_io_stats", "false"},
+ {"purge_redundant_kvs_while_flush", "false"},
+ {"inplace_update_num_locks", "25"},
+ {"memtable_prefix_bloom_size_ratio", "0.26"},
+ {"memtable_whole_key_filtering", "true"},
+ {"memtable_huge_page_size", "28"},
+ {"bloom_locality", "29"},
+ {"max_successive_merges", "30"},
+ {"min_partial_merge_operands", "31"},
+ {"prefix_extractor", "fixed:31"},
+ {"experimental_mempurge_threshold", "0.003"},
+ {"optimize_filters_for_hits", "true"},
+ {"enable_blob_files", "true"},
+ {"min_blob_size", "1K"},
+ {"blob_file_size", "1G"},
+ {"blob_compression_type", "kZSTD"},
+ {"enable_blob_garbage_collection", "true"},
+ {"blob_garbage_collection_age_cutoff", "0.5"},
+ {"blob_garbage_collection_force_threshold", "0.75"},
+ {"blob_compaction_readahead_size", "256K"},
+ {"blob_file_starting_level", "1"},
+ {"prepopulate_blob_cache", "kDisable"},
+ {"last_level_temperature", "kWarm"},
+ };
+
+ std::unordered_map<std::string, std::string> db_options_map = {
+ {"create_if_missing", "false"},
+ {"create_missing_column_families", "true"},
+ {"error_if_exists", "false"},
+ {"paranoid_checks", "true"},
+ {"track_and_verify_wals_in_manifest", "true"},
+ {"verify_sst_unique_id_in_manifest", "true"},
+ {"max_open_files", "32"},
+ {"max_total_wal_size", "33"},
+ {"use_fsync", "true"},
+ {"db_log_dir", "/db_log_dir"},
+ {"wal_dir", "/wal_dir"},
+ {"delete_obsolete_files_period_micros", "34"},
+ {"max_background_compactions", "35"},
+ {"max_background_flushes", "36"},
+ {"max_log_file_size", "37"},
+ {"log_file_time_to_roll", "38"},
+ {"keep_log_file_num", "39"},
+ {"recycle_log_file_num", "5"},
+ {"max_manifest_file_size", "40"},
+ {"table_cache_numshardbits", "41"},
+ {"WAL_ttl_seconds", "43"},
+ {"WAL_size_limit_MB", "44"},
+ {"manifest_preallocation_size", "45"},
+ {"allow_mmap_reads", "true"},
+ {"allow_mmap_writes", "false"},
+ {"use_direct_reads", "false"},
+ {"use_direct_io_for_flush_and_compaction", "false"},
+ {"is_fd_close_on_exec", "true"},
+ {"skip_log_error_on_recovery", "false"},
+ {"stats_dump_period_sec", "46"},
+ {"stats_persist_period_sec", "57"},
+ {"persist_stats_to_disk", "false"},
+ {"stats_history_buffer_size", "69"},
+ {"advise_random_on_open", "true"},
+ {"use_adaptive_mutex", "false"},
+ {"compaction_readahead_size", "100"},
+ {"random_access_max_buffer_size", "3145728"},
+ {"writable_file_max_buffer_size", "314159"},
+ {"bytes_per_sync", "47"},
+ {"wal_bytes_per_sync", "48"},
+ {"strict_bytes_per_sync", "true"},
+ {"preserve_deletes", "false"},
+ };
+
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ConfigOptions exact, loose;
+ exact.input_strings_escaped = false;
+ exact.ignore_unknown_options = false;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
+
+ loose.input_strings_escaped = false;
+ loose.ignore_unknown_options = true;
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
+ ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999);
+ ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
+ ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
+ ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
+ ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
+ ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 2u);
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_buffer_bytes, 100u);
+ ASSERT_EQ(new_cf_opt.compression_opts.use_zstd_dict_trainer, false);
+ ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 10u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads,
+ CompressionOptions().parallel_threads);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer,
+ CompressionOptions().use_zstd_dict_trainer);
+ ASSERT_EQ(new_cf_opt.num_levels, 8);
+ ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8);
+ ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9);
+ ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10);
+ ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12));
+ ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
+ ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18);
+ ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21);
+ ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
+ ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
+ ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
+ ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
+ ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
+ static_cast<uint64_t>(23));
+ ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
+ static_cast<uint64_t>(24));
+ ASSERT_EQ(new_cf_opt.inplace_update_support, true);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
+ ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
+ ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true);
+ ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
+ ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
+ ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
+ ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
+ ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.FixedPrefix.31");
+ ASSERT_EQ(new_cf_opt.experimental_mempurge_threshold, 0.003);
+ ASSERT_EQ(new_cf_opt.enable_blob_files, true);
+ ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
+ ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
+ ASSERT_EQ(new_cf_opt.blob_compression_type, kZSTD);
+ ASSERT_EQ(new_cf_opt.enable_blob_garbage_collection, true);
+ ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
+ ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75);
+ ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
+ ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1);
+ ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable);
+ ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm);
+ ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
+
+ cf_options_map["write_buffer_size"] = "hello";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ cf_options_map["write_buffer_size"] = "1";
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+
+ cf_options_map["unknown_option"] = "1";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // ignore_unknown_options=true;input_strings_escaped=false
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(loose, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(loose, base_cf_opt, new_cf_opt));
+ ASSERT_NOK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ DBOptions base_db_opt;
+ DBOptions new_db_opt;
+ ASSERT_OK(
+ GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.create_missing_column_families, true);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.paranoid_checks, true);
+ ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
+ ASSERT_EQ(new_db_opt.verify_sst_unique_id_in_manifest, true);
+ ASSERT_EQ(new_db_opt.max_open_files, 32);
+ ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
+ ASSERT_EQ(new_db_opt.use_fsync, true);
+ ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir");
+ ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir");
+ ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros,
+ static_cast<uint64_t>(34));
+ ASSERT_EQ(new_db_opt.max_background_compactions, 35);
+ ASSERT_EQ(new_db_opt.max_background_flushes, 36);
+ ASSERT_EQ(new_db_opt.max_log_file_size, 37U);
+ ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U);
+ ASSERT_EQ(new_db_opt.keep_log_file_num, 39U);
+ ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U);
+ ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40));
+ ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41);
+ ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43));
+ ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44));
+ ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U);
+ ASSERT_EQ(new_db_opt.allow_mmap_reads, true);
+ ASSERT_EQ(new_db_opt.allow_mmap_writes, false);
+ ASSERT_EQ(new_db_opt.use_direct_reads, false);
+ ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false);
+ ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true);
+ ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U);
+ ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U);
+ ASSERT_EQ(new_db_opt.persist_stats_to_disk, false);
+ ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U);
+ ASSERT_EQ(new_db_opt.advise_random_on_open, true);
+ ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
+ ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
+ ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
+ ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
+ ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
+ ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
+ ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true);
+
+ db_options_map["max_open_files"] = "hello";
+ Status s =
+ GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
+
+ // unknow options should fail parsing without ignore_unknown_options = true
+ db_options_map["unknown_db_option"] = "1";
+ s = GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+
+ ASSERT_OK(
+ GetDBOptionsFromMap(loose, base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
+ ASSERT_NOK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+}
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in
+ // ROCKSDB_LITE
+TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
+ base_cf_opt.table_factory.reset();
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, "",
+ &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=5", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
+ ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=6;", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, " write_buffer_size = 7 ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, " write_buffer_size = 8 ; ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=11; max_write_buffer_number = 12 ;", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
+ // Wrong name "max_write_buffer_number_"
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number_=14;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ // Comparator from object registry
+ std::string kCompName = "reverse_comp";
+ ObjectLibrary::Default()->AddFactory<const Comparator>(
+ kCompName,
+ [](const std::string& /*name*/,
+ std::unique_ptr<const Comparator>* /*guard*/,
+ std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "comparator=" + kCompName + ";",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator());
+
+ // MergeOperator from object registry
+ std::unique_ptr<BytesXOROperator> bxo(new BytesXOROperator());
+ std::string kMoName = bxo->Name();
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "merge_operator=" + kMoName + ";",
+ &new_cf_opt));
+ ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name()));
+
+ // Wrong key/value pair
+ Status s = GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ // Error Parsing value
+ s = GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ // Missing option name
+ s = GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=13; =100;", &new_cf_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ const uint64_t kilo = 1024UL;
+ const uint64_t mega = 1024 * kilo;
+ const uint64_t giga = 1024 * mega;
+ const uint64_t tera = 1024 * giga;
+
+ // Units (k)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo);
+ // Units (m)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "max_write_buffer_number=16m;inplace_update_num_locks=17M", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega);
+ // Units (g)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=18g;prefix_extractor=capped:8;"
+ "arena_block_size=19G",
+ &new_cf_opt));
+
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
+ ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.CappedPrefix.8");
+
+ // Units (t)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=20t;arena_block_size=21T",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
+
+ // Nested block based table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Last one
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;}",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Mismatch curly braces
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={{{block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ // Unexpected chars after closing curly brace
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa;"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ // Invalid block based table option
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={xx_block_size=4;}",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "optimize_filters_for_hits=true",
+ &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "optimize_filters_for_hits=false",
+ &new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "optimize_filters_for_hits=junk",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
+
+ // Nested plain table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+
+ // memtable factory
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "memtable=skip_list:10;arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
+ ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
+
+ // blob cache
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "blob_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};",
+ &new_cf_opt));
+ ASSERT_NE(new_cf_opt.blob_cache, nullptr);
+ ASSERT_EQ(new_cf_opt.blob_cache->GetCapacity(), 1024UL * 1024UL);
+ ASSERT_EQ(static_cast<ShardedCacheBase*>(new_cf_opt.blob_cache.get())
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_cf_opt.blob_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(static_cast<LRUCache*>(new_cf_opt.blob_cache.get())
+ ->GetHighPriPoolRatio(),
+ 0.5);
+}
+
+TEST_F(OptionsTest, CompressionOptionsFromString) {
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ConfigOptions config_options;
+ std::string opts_str;
+ config_options.ignore_unknown_options = false;
+ CompressionOptions dflt;
+ // Test with some optional values removed....
+ ASSERT_OK(
+ GetColumnFamilyOptionsFromString(config_options, ColumnFamilyOptions(),
+ "compression_opts=3:4:5; "
+ "bottommost_compression_opts=4:5:6:7",
+ &base_cf_opt));
+ ASSERT_EQ(base_cf_opt.compression_opts.window_bits, 3);
+ ASSERT_EQ(base_cf_opt.compression_opts.level, 4);
+ ASSERT_EQ(base_cf_opt.compression_opts.strategy, 5);
+ ASSERT_EQ(base_cf_opt.compression_opts.max_dict_bytes, dflt.max_dict_bytes);
+ ASSERT_EQ(base_cf_opt.compression_opts.zstd_max_train_bytes,
+ dflt.zstd_max_train_bytes);
+ ASSERT_EQ(base_cf_opt.compression_opts.parallel_threads,
+ dflt.parallel_threads);
+ ASSERT_EQ(base_cf_opt.compression_opts.enabled, dflt.enabled);
+ ASSERT_EQ(base_cf_opt.compression_opts.use_zstd_dict_trainer,
+ dflt.use_zstd_dict_trainer);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.window_bits, 4);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.level, 5);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.strategy, 6);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.zstd_max_train_bytes,
+ dflt.zstd_max_train_bytes);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.parallel_threads,
+ dflt.parallel_threads);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.enabled, dflt.enabled);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer,
+ dflt.use_zstd_dict_trainer);
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(),
+ "compression_opts=4:5:6:7:8:9:true:10:false; "
+ "bottommost_compression_opts=5:6:7:8:9:false",
+ &base_cf_opt));
+ ASSERT_EQ(base_cf_opt.compression_opts.window_bits, 4);
+ ASSERT_EQ(base_cf_opt.compression_opts.level, 5);
+ ASSERT_EQ(base_cf_opt.compression_opts.strategy, 6);
+ ASSERT_EQ(base_cf_opt.compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(base_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
+ ASSERT_EQ(base_cf_opt.compression_opts.parallel_threads, 9u);
+ ASSERT_EQ(base_cf_opt.compression_opts.enabled, true);
+ ASSERT_EQ(base_cf_opt.compression_opts.max_dict_buffer_bytes, 10u);
+ ASSERT_EQ(base_cf_opt.compression_opts.use_zstd_dict_trainer, false);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.parallel_threads,
+ dflt.parallel_threads);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.enabled, false);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer,
+ dflt.use_zstd_dict_trainer);
+
+ ASSERT_OK(
+ GetStringFromColumnFamilyOptions(config_options, base_cf_opt, &opts_str));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(), opts_str, &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
+ ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 9u);
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
+ ASSERT_EQ(base_cf_opt.compression_opts.max_dict_buffer_bytes, 10u);
+ ASSERT_EQ(base_cf_opt.compression_opts.use_zstd_dict_trainer, false);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads,
+ dflt.parallel_threads);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, false);
+ ASSERT_EQ(base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer,
+ dflt.use_zstd_dict_trainer);
+
+ // Test as struct values
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(),
+ "compression_opts={window_bits=5; level=6; strategy=7; max_dict_bytes=8;"
+ "zstd_max_train_bytes=9;parallel_threads=10;enabled=true;use_zstd_dict_"
+ "trainer=false}; "
+ "bottommost_compression_opts={window_bits=4; level=5; strategy=6;"
+ " max_dict_bytes=7;zstd_max_train_bytes=8;parallel_threads=9;"
+ "enabled=false;use_zstd_dict_trainer=true}; ",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 5);
+ ASSERT_EQ(new_cf_opt.compression_opts.level, 6);
+ ASSERT_EQ(new_cf_opt.compression_opts.strategy, 7);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 9u);
+ ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 10u);
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.compression_opts.use_zstd_dict_trainer, false);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 4);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 5);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 6);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, 9u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, false);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer, true);
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "compression_opts={window_bits=4; strategy=5;};"
+ "bottommost_compression_opts={level=6; strategy=7;}",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_cf_opt.compression_opts.strategy, 5);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
+
+ ASSERT_EQ(new_cf_opt.compression_opts.level,
+ base_cf_opt.compression_opts.level);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes,
+ base_cf_opt.compression_opts.max_dict_bytes);
+ ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes,
+ base_cf_opt.compression_opts.zstd_max_train_bytes);
+ ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads,
+ base_cf_opt.compression_opts.parallel_threads);
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled,
+ base_cf_opt.compression_opts.enabled);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits,
+ base_cf_opt.bottommost_compression_opts.window_bits);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes,
+ base_cf_opt.bottommost_compression_opts.max_dict_bytes);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes,
+ base_cf_opt.bottommost_compression_opts.zstd_max_train_bytes);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads,
+ base_cf_opt.bottommost_compression_opts.parallel_threads);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled,
+ base_cf_opt.bottommost_compression_opts.enabled);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer,
+ base_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer);
+
+ // Test a few individual struct values
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "compression_opts.enabled=false; "
+ "bottommost_compression_opts.enabled=true; ",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled, false);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true);
+
+ // Now test some illegal values
+ ConfigOptions ignore;
+ ignore.ignore_unknown_options = true;
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(),
+ "compression_opts=5:6:7:8:9:x:false", &base_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ ignore, ColumnFamilyOptions(), "compression_opts=5:6:7:8:9:x:false",
+ &base_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(),
+ "compression_opts=1:2:3:4:5:6:true:8", &base_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ ignore, ColumnFamilyOptions(), "compression_opts=1:2:3:4:5:6:true:8",
+ &base_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(),
+ "compression_opts=1:2:3:4:5:6:true:8:9", &base_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ ignore, ColumnFamilyOptions(), "compression_opts=1:2:3:4:5:6:true:8:9",
+ &base_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(), "compression_opts={unknown=bad;}",
+ &base_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(ignore, ColumnFamilyOptions(),
+ "compression_opts={unknown=bad;}",
+ &base_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, ColumnFamilyOptions(), "compression_opts.unknown=bad",
+ &base_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(ignore, ColumnFamilyOptions(),
+ "compression_opts.unknown=bad",
+ &base_cf_opt));
+}
+
+TEST_F(OptionsTest, OldInterfaceTest) {
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ConfigOptions exact;
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "write_buffer_size=18;prefix_extractor=capped:8;"
+ "arena_block_size=19",
+ &new_cf_opt));
+
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 18);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 19);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
+
+ // And with a bad option
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={xx_block_size=4;}",
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ std::unordered_map<std::string, std::string> cf_options_map = {
+ {"write_buffer_size", "1"},
+ {"max_write_buffer_number", "2"},
+ {"min_write_buffer_number_to_merge", "3"},
+ };
+ ASSERT_OK(
+ GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, &new_cf_opt));
+ cf_options_map["unknown_option"] = "1";
+ ASSERT_NOK(
+ GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
+ &new_cf_opt, true, true));
+
+ DBOptions base_db_opt;
+ DBOptions new_db_opt;
+ std::unordered_map<std::string, std::string> db_options_map = {
+ {"create_if_missing", "false"},
+ {"create_missing_column_families", "true"},
+ {"error_if_exists", "false"},
+ {"paranoid_checks", "true"},
+ {"track_and_verify_wals_in_manifest", "true"},
+ {"verify_sst_unique_id_in_manifest", "true"},
+ {"max_open_files", "32"},
+ };
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.create_missing_column_families, true);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.paranoid_checks, true);
+ ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
+ ASSERT_EQ(new_db_opt.verify_sst_unique_id_in_manifest, true);
+ ASSERT_EQ(new_db_opt.max_open_files, 32);
+ db_options_map["unknown_option"] = "1";
+ Status s = GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt, true,
+ true));
+ ASSERT_OK(GetDBOptionsFromString(
+ base_db_opt,
+ "create_if_missing=false;error_if_exists=false;max_open_files=42;",
+ &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.max_open_files, 42);
+ s = GetDBOptionsFromString(
+ base_db_opt,
+ "create_if_missing=false;error_if_exists=false;max_open_files=42;"
+ "unknown_option=1;",
+ &new_db_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+}
+
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported
+TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
+ BlockBasedTableOptions table_opt;
+ BlockBasedTableOptions new_opt;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+ config_options.ignore_unsupported_options = false;
+
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kHashSearch;"
+ "checksum=kxxHash;"
+ "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
+ "block_size_deviation=8;block_restart_interval=4;"
+ "format_version=5;whole_key_filtering=1;"
+ "filter_policy=bloomfilter:4.567:false;detect_filter_construct_"
+ "corruption=true;"
+ // A bug caused read_amp_bytes_per_bit to be a large integer in OPTIONS
+ // file generated by 6.10 to 6.14. Though bug is fixed in these releases,
+ // we need to handle the case of loading OPTIONS file generated before the
+ // fix.
+ "read_amp_bytes_per_bit=17179869185;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
+ ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL);
+ ASSERT_EQ(new_opt.block_size, 1024UL);
+ ASSERT_EQ(new_opt.block_size_deviation, 8);
+ ASSERT_EQ(new_opt.block_restart_interval, 4);
+ ASSERT_EQ(new_opt.format_version, 5U);
+ ASSERT_EQ(new_opt.whole_key_filtering, true);
+ ASSERT_EQ(new_opt.detect_filter_construct_corruption, true);
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ auto bfp = new_opt.filter_policy->CheckedCast<BloomFilterPolicy>();
+ ASSERT_NE(bfp, nullptr);
+ EXPECT_EQ(bfp->GetMillibitsPerKey(), 4567);
+ EXPECT_EQ(bfp->GetWholeBitsPerKey(), 5);
+ // Verify that only the lower 32bits are stored in
+ // new_opt.read_amp_bytes_per_bit.
+ EXPECT_EQ(1U, new_opt.read_amp_bytes_per_bit);
+
+ // unknown option
+ Status s = GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
+ "bad_option=1",
+ &new_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_EQ(static_cast<bool>(table_opt.cache_index_and_filter_blocks),
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized index type
+ s = GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX", &new_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized checksum type
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "cache_index_and_filter_blocks=1;checksum=kxxHashXX", &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized filter policy name
+ s = GetBlockBasedTableOptionsFromString(config_options, table_opt,
+ "filter_policy=bloomfilterxx:4:true",
+ &new_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ // missing bits per key
+ s = GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=bloomfilter", &new_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ // Used to be rejected, now accepted
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=bloomfilter:4", &new_opt));
+ bfp = dynamic_cast<const BloomFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(bfp->GetMillibitsPerKey(), 4000);
+ EXPECT_EQ(bfp->GetWholeBitsPerKey(), 4);
+
+ // use_block_based_builder=true now ignored in public API (same as false)
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=bloomfilter:4:true", &new_opt));
+ bfp = dynamic_cast<const BloomFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(bfp->GetMillibitsPerKey(), 4000);
+ EXPECT_EQ(bfp->GetWholeBitsPerKey(), 4);
+
+ // Test configuring using other internal names
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "filter_policy=rocksdb.internal.LegacyBloomFilter:3", &new_opt));
+ auto builtin =
+ dynamic_cast<const BuiltinFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(builtin->GetId(), "rocksdb.internal.LegacyBloomFilter:3");
+
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "filter_policy=rocksdb.internal.FastLocalBloomFilter:1.234", &new_opt));
+ builtin =
+ dynamic_cast<const BuiltinFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(builtin->GetId(), "rocksdb.internal.FastLocalBloomFilter:1.234");
+
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "filter_policy=rocksdb.internal.Standard128RibbonFilter:1.234",
+ &new_opt));
+ builtin =
+ dynamic_cast<const BuiltinFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(builtin->GetId(), "rocksdb.internal.Standard128RibbonFilter:1.234");
+
+ // Ribbon filter policy (no Bloom hybrid)
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=ribbonfilter:5.678:-1;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ auto rfp =
+ dynamic_cast<const RibbonFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(rfp->GetMillibitsPerKey(), 5678);
+ EXPECT_EQ(rfp->GetBloomBeforeLevel(), -1);
+
+ // Ribbon filter policy (default Bloom hybrid)
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=ribbonfilter:6.789;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ rfp = dynamic_cast<const RibbonFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(rfp->GetMillibitsPerKey(), 6789);
+ EXPECT_EQ(rfp->GetBloomBeforeLevel(), 0);
+
+ // Ribbon filter policy (custom Bloom hybrid)
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=ribbonfilter:6.789:5;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ rfp = dynamic_cast<const RibbonFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(rfp->GetMillibitsPerKey(), 6789);
+ EXPECT_EQ(rfp->GetBloomBeforeLevel(), 5);
+
+ // Check block cache options are overwritten when specified
+ // in new format as a struct.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set only block cache capacity. Check other values are
+ // reset to default values.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "block_cache={capacity=2M};"
+ "block_cache_compressed={capacity=2M}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(
+ std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ GetDefaultCacheShardBits(new_opt.block_cache_compressed->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={num_shard_bits=5;"
+ "high_pri_pool_ratio=0.0;}",
+ &new_opt));
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ 5);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ 5);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.0);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=rocksdb.BloomFilter:1.234",
+ &new_opt));
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ ASSERT_TRUE(
+ new_opt.filter_policy->IsInstanceOf(BloomFilterPolicy::kClassName()));
+ ASSERT_TRUE(
+ new_opt.filter_policy->IsInstanceOf(BloomFilterPolicy::kNickName()));
+
+ // Ribbon filter policy alternative name
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt, "filter_policy=rocksdb.RibbonFilter:6.789:5;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ ASSERT_TRUE(
+ new_opt.filter_policy->IsInstanceOf(RibbonFilterPolicy::kClassName()));
+ ASSERT_TRUE(
+ new_opt.filter_policy->IsInstanceOf(RibbonFilterPolicy::kNickName()));
+}
+#endif // !ROCKSDB_LITE
+
+
+#ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported
+TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
+ PlainTableOptions table_opt;
+ PlainTableOptions new_opt;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetPlainTableOptionsFromString(
+ config_options, table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
+ "full_scan_mode=true;store_index_in_file=true",
+ &new_opt));
+ ASSERT_EQ(new_opt.user_key_len, 66u);
+ ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
+ ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
+ ASSERT_EQ(new_opt.index_sparseness, 8);
+ ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
+ ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
+ ASSERT_TRUE(new_opt.full_scan_mode);
+ ASSERT_TRUE(new_opt.store_index_in_file);
+
+ // unknown option
+ Status s = GetPlainTableOptionsFromString(
+ config_options, table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "bad_option=1",
+ &new_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ // unrecognized EncodingType
+ s = GetPlainTableOptionsFromString(
+ config_options, table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "encoding_type=kPrefixXX",
+ &new_opt);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+}
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported
+TEST_F(OptionsTest, GetMemTableRepFactoryFromString) {
+ std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr;
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000",
+ &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist",
+ &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000",
+ &new_mem_factory));
+ ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory));
+ ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory));
+ ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory");
+ ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt",
+ &new_mem_factory));
+
+ ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory));
+ // CuckooHash memtable is already removed.
+ ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory));
+
+ ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory));
+}
+#endif // !ROCKSDB_LITE
+
+TEST_F(OptionsTest, MemTableRepFactoryCreateFromString) {
+ std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr;
+ ConfigOptions config_options;
+ config_options.ignore_unsupported_options = false;
+ config_options.ignore_unknown_options = false;
+
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list",
+ &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list:16",
+ &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("skip_list"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("SkipListFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "skip_list:16:invalid_opt", &new_mem_factory));
+
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "invalid_opt=10", &new_mem_factory));
+
+ // Test a reset
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "",
+ &new_mem_factory));
+ ASSERT_EQ(new_mem_factory, nullptr);
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "invalid_opt=10", &new_mem_factory));
+
+#ifndef ROCKSDB_LITE
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "id=skip_list; lookahead=32", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "prefix_hash",
+ &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "prefix_hash:1000", &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("prefix_hash"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashSkipListRepFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "prefix_hash:1000:invalid_opt", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=prefix_hash; bucket_count=32; skiplist_height=64; "
+ "branching_factor=16",
+ &new_mem_factory));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=prefix_hash; bucket_count=32; skiplist_height=64; "
+ "branching_factor=16; invalid=unknown",
+ &new_mem_factory));
+
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "hash_linkedlist", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "hash_linkedlist:1000", &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "HashLinkListRepFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("hash_linkedlist"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashLinkListRepFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "hash_linkedlist:1000:invalid_opt", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; "
+ "logging_threshold=12; log_when_flash=true",
+ &new_mem_factory));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; "
+ "logging_threshold=12; log_when_flash=true; invalid=unknown",
+ &new_mem_factory));
+
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector",
+ &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector:1024",
+ &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "VectorRepFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("vector"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("VectorRepFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "vector:1024:invalid_opt", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "id=vector; count=42", &new_mem_factory));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "id=vector; invalid=unknown", &new_mem_factory));
+#endif // ROCKSDB_LITE
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo",
+ &new_mem_factory));
+ // CuckooHash memtable is already removed.
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo:1024",
+ &new_mem_factory));
+
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "bad_factory",
+ &new_mem_factory));
+}
+
+#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
+class CustomEnv : public EnvWrapper {
+ public:
+ explicit CustomEnv(Env* _target) : EnvWrapper(_target) {}
+ static const char* kClassName() { return "CustomEnv"; }
+ const char* Name() const override { return kClassName(); }
+};
+
+TEST_F(OptionsTest, GetOptionsFromStringTest) {
+ Options base_options, new_options;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
+ base_options.write_buffer_size = 20;
+ base_options.min_write_buffer_number_to_merge = 15;
+ BlockBasedTableOptions block_based_table_options;
+ block_based_table_options.cache_index_and_filter_blocks = true;
+ base_options.table_factory.reset(
+ NewBlockBasedTableFactory(block_based_table_options));
+
+ // Register an Env with object registry.
+ ObjectLibrary::Default()->AddFactory<Env>(
+ CustomEnv::kClassName(),
+ [](const std::string& /*name*/, std::unique_ptr<Env>* /*env_guard*/,
+ std::string* /* errmsg */) {
+ static CustomEnv env(Env::Default());
+ return &env;
+ });
+
+ ASSERT_OK(GetOptionsFromString(
+ config_options, base_options,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;"
+ "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files="
+ "1;"
+ "rate_limiter_bytes_per_sec=1024;env=CustomEnv",
+ &new_options));
+
+ ASSERT_EQ(new_options.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_options.compression_opts.level, 5);
+ ASSERT_EQ(new_options.compression_opts.strategy, 6);
+ ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.parallel_threads, 1u);
+ ASSERT_EQ(new_options.compression_opts.enabled, false);
+ ASSERT_EQ(new_options.compression_opts.use_zstd_dict_trainer, true);
+ ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption);
+ ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_options.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.parallel_threads, 1u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
+ ASSERT_EQ(new_options.bottommost_compression_opts.use_zstd_dict_trainer,
+ true);
+ ASSERT_EQ(new_options.write_buffer_size, 10U);
+ ASSERT_EQ(new_options.max_write_buffer_number, 16);
+ const auto new_bbto =
+ new_options.table_factory->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(new_bbto, nullptr);
+ ASSERT_EQ(new_bbto->block_cache->GetCapacity(), 1U << 20);
+ ASSERT_EQ(new_bbto->block_size, 4U);
+ // don't overwrite block based table options
+ ASSERT_TRUE(new_bbto->cache_index_and_filter_blocks);
+
+ ASSERT_EQ(new_options.create_if_missing, true);
+ ASSERT_EQ(new_options.max_open_files, 1);
+ ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
+ Env* newEnv = new_options.env;
+ ASSERT_OK(Env::LoadEnv(CustomEnv::kClassName(), &newEnv));
+ ASSERT_EQ(newEnv, new_options.env);
+
+ config_options.ignore_unknown_options = false;
+ // Test a bad value for a DBOption returns a failure
+ base_options.dump_malloc_stats = false;
+ base_options.write_buffer_size = 1024;
+ Options bad_options = new_options;
+ Status s = GetOptionsFromString(config_options, base_options,
+ "create_if_missing=XX;dump_malloc_stats=true",
+ &bad_options);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_EQ(bad_options.dump_malloc_stats, false);
+
+ bad_options = new_options;
+ s = GetOptionsFromString(config_options, base_options,
+ "write_buffer_size=XX;dump_malloc_stats=true",
+ &bad_options);
+ ASSERT_NOK(s);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ ASSERT_EQ(bad_options.dump_malloc_stats, false);
+
+ // Test a bad value for a TableFactory Option returns a failure
+ bad_options = new_options;
+ s = GetOptionsFromString(config_options, base_options,
+ "write_buffer_size=16;dump_malloc_stats=true"
+ "block_based_table_factory={block_size=XX;};",
+ &bad_options);
+ ASSERT_TRUE(s.IsInvalidArgument());
+ ASSERT_EQ(bad_options.dump_malloc_stats, false);
+ ASSERT_EQ(bad_options.write_buffer_size, 1024);
+
+ config_options.ignore_unknown_options = true;
+ ASSERT_OK(GetOptionsFromString(config_options, base_options,
+ "create_if_missing=XX;dump_malloc_stats=true;"
+ "write_buffer_size=XX;"
+ "block_based_table_factory={block_size=XX;};",
+ &bad_options));
+ ASSERT_EQ(bad_options.create_if_missing, base_options.create_if_missing);
+ ASSERT_EQ(bad_options.dump_malloc_stats, true);
+ ASSERT_EQ(bad_options.write_buffer_size, base_options.write_buffer_size);
+
+ // Test the old interface
+ ASSERT_OK(GetOptionsFromString(
+ base_options,
+ "write_buffer_size=22;max_write_buffer_number=33;max_open_files=44;",
+ &new_options));
+ ASSERT_EQ(new_options.write_buffer_size, 22U);
+ ASSERT_EQ(new_options.max_write_buffer_number, 33);
+ ASSERT_EQ(new_options.max_open_files, 44);
+}
+
+TEST_F(OptionsTest, DBOptionsSerialization) {
+ Options base_options, new_options;
+ Random rnd(301);
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
+ // Phase 1: Make big change in base_options
+ test::RandomInitDBOptions(&base_options, &rnd);
+
+ // Phase 2: obtain a string from base_option
+ std::string base_options_file_content;
+ ASSERT_OK(GetStringFromDBOptions(config_options, base_options,
+ &base_options_file_content));
+
+ // Phase 3: Set new_options from the derived string and expect
+ // new_options == base_options
+ ASSERT_OK(GetDBOptionsFromString(config_options, DBOptions(),
+ base_options_file_content, &new_options));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_options,
+ new_options));
+}
+
+TEST_F(OptionsTest, OptionsComposeDecompose) {
+ // build an Options from DBOptions + CFOptions, then decompose it to verify
+ // we get same constituent options.
+ DBOptions base_db_opts;
+ ColumnFamilyOptions base_cf_opts;
+ ConfigOptions
+ config_options; // Use default for ignore(false) and check (exact)
+ config_options.input_strings_escaped = false;
+
+ Random rnd(301);
+ test::RandomInitDBOptions(&base_db_opts, &rnd);
+ test::RandomInitCFOptions(&base_cf_opts, base_db_opts, &rnd);
+
+ Options base_opts(base_db_opts, base_cf_opts);
+ DBOptions new_db_opts(base_opts);
+ ColumnFamilyOptions new_cf_opts(base_opts);
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_db_opts,
+ new_db_opts));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opts,
+ new_cf_opts));
+ delete new_cf_opts.compaction_filter;
+}
+
+TEST_F(OptionsTest, DBOptionsComposeImmutable) {
+ // Build a DBOptions from an Immutable/Mutable one and verify that
+ // we get same constituent options.
+ ConfigOptions config_options;
+ Random rnd(301);
+ DBOptions base_opts, new_opts;
+ test::RandomInitDBOptions(&base_opts, &rnd);
+ MutableDBOptions m_opts(base_opts);
+ ImmutableDBOptions i_opts(base_opts);
+ new_opts = BuildDBOptions(i_opts, m_opts);
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_opts,
+ new_opts));
+}
+
+TEST_F(OptionsTest, GetMutableDBOptions) {
+ Random rnd(228);
+ DBOptions base_opts;
+ std::string opts_str;
+ std::unordered_map<std::string, std::string> opts_map;
+ ConfigOptions config_options;
+
+ test::RandomInitDBOptions(&base_opts, &rnd);
+ ImmutableDBOptions i_opts(base_opts);
+ MutableDBOptions m_opts(base_opts);
+ MutableDBOptions new_opts;
+ ASSERT_OK(GetStringFromMutableDBOptions(config_options, m_opts, &opts_str));
+ ASSERT_OK(StringToMap(opts_str, &opts_map));
+ ASSERT_OK(GetMutableDBOptionsFromStrings(m_opts, opts_map, &new_opts));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
+ config_options, base_opts, BuildDBOptions(i_opts, new_opts)));
+}
+
+TEST_F(OptionsTest, CFOptionsComposeImmutable) {
+ // Build a DBOptions from an Immutable/Mutable one and verify that
+ // we get same constituent options.
+ ConfigOptions config_options;
+ Random rnd(301);
+ ColumnFamilyOptions base_opts, new_opts;
+ DBOptions dummy; // Needed to create ImmutableCFOptions
+ test::RandomInitCFOptions(&base_opts, dummy, &rnd);
+ MutableCFOptions m_opts(base_opts);
+ ImmutableCFOptions i_opts(base_opts);
+ UpdateColumnFamilyOptions(i_opts, &new_opts);
+ UpdateColumnFamilyOptions(m_opts, &new_opts);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_opts,
+ new_opts));
+ delete new_opts.compaction_filter;
+}
+
+TEST_F(OptionsTest, GetMutableCFOptions) {
+ Random rnd(228);
+ ColumnFamilyOptions base, copy;
+ std::string opts_str;
+ std::unordered_map<std::string, std::string> opts_map;
+ ConfigOptions config_options;
+ DBOptions dummy; // Needed to create ImmutableCFOptions
+
+ test::RandomInitCFOptions(&base, dummy, &rnd);
+ ColumnFamilyOptions result;
+ MutableCFOptions m_opts(base), new_opts;
+
+ ASSERT_OK(GetStringFromMutableCFOptions(config_options, m_opts, &opts_str));
+ ASSERT_OK(StringToMap(opts_str, &opts_map));
+ ASSERT_OK(GetMutableOptionsFromStrings(m_opts, opts_map, nullptr, &new_opts));
+ UpdateColumnFamilyOptions(ImmutableCFOptions(base), &copy);
+ UpdateColumnFamilyOptions(new_opts, &copy);
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base, copy));
+ delete copy.compaction_filter;
+}
+
+TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
+ Options options;
+ ColumnFamilyOptions base_opt, new_opt;
+ Random rnd(302);
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+
+ // Phase 1: randomly assign base_opt
+ // custom type options
+ test::RandomInitCFOptions(&base_opt, options, &rnd);
+
+ // Phase 2: obtain a string from base_opt
+ std::string base_options_file_content;
+ ASSERT_OK(GetStringFromColumnFamilyOptions(config_options, base_opt,
+ &base_options_file_content));
+
+ // Phase 3: Set new_opt from the derived string and expect
+ // new_opt == base_opt
+ ASSERT_OK(
+ GetColumnFamilyOptionsFromString(config_options, ColumnFamilyOptions(),
+ base_options_file_content, &new_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt));
+ if (base_opt.compaction_filter) {
+ delete base_opt.compaction_filter;
+ }
+}
+
+TEST_F(OptionsTest, CheckBlockBasedTableOptions) {
+ ColumnFamilyOptions cf_opts;
+ DBOptions db_opts;
+ ConfigOptions config_opts;
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_opts, cf_opts, "prefix_extractor=capped:8", &cf_opts));
+ ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable",
+ &cf_opts.table_factory));
+ ASSERT_NE(cf_opts.table_factory.get(), nullptr);
+ ASSERT_TRUE(cf_opts.table_factory->IsInstanceOf(
+ TableFactory::kBlockBasedTableName()));
+ auto bbto = cf_opts.table_factory->GetOptions<BlockBasedTableOptions>();
+ ASSERT_OK(cf_opts.table_factory->ConfigureFromString(
+ config_opts,
+ "block_cache={capacity=1M;num_shard_bits=4;};"
+ "block_size_deviation=101;"
+ "block_restart_interval=0;"
+ "index_block_restart_interval=5;"
+ "partition_filters=true;"
+ "index_type=kHashSearch;"
+ "no_block_cache=1;"));
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_EQ(bbto->block_cache.get(), nullptr);
+ ASSERT_EQ(bbto->block_size_deviation, 0);
+ ASSERT_EQ(bbto->block_restart_interval, 1);
+ ASSERT_EQ(bbto->index_block_restart_interval, 1);
+ ASSERT_FALSE(bbto->partition_filters);
+ ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable",
+ &cf_opts.table_factory));
+ bbto = cf_opts.table_factory->GetOptions<BlockBasedTableOptions>();
+
+ ASSERT_OK(cf_opts.table_factory->ConfigureFromString(config_opts,
+ "no_block_cache=0;"));
+ ASSERT_NE(bbto->block_cache.get(), nullptr);
+ ASSERT_OK(cf_opts.table_factory->ValidateOptions(db_opts, cf_opts));
+}
+
+TEST_F(OptionsTest, MutableTableOptions) {
+ ConfigOptions config_options;
+ std::shared_ptr<TableFactory> bbtf;
+ bbtf.reset(NewBlockBasedTableFactory());
+ auto bbto = bbtf->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_OK(bbtf->ConfigureOption(config_options, "block_align", "true"));
+ ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024"));
+ ASSERT_EQ(bbto->block_align, true);
+ ASSERT_EQ(bbto->block_size, 1024);
+ ASSERT_OK(bbtf->PrepareOptions(config_options));
+ config_options.mutable_options_only = true;
+ ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024"));
+ ASSERT_EQ(bbto->block_align, true);
+ ASSERT_NOK(bbtf->ConfigureOption(config_options, "block_align", "false"));
+ ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "2048"));
+ ASSERT_EQ(bbto->block_align, true);
+ ASSERT_EQ(bbto->block_size, 2048);
+
+ ColumnFamilyOptions cf_opts;
+ cf_opts.table_factory = bbtf;
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, cf_opts, "block_based_table_factory.block_align=false",
+ &cf_opts));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, cf_opts, "block_based_table_factory.block_size=8192",
+ &cf_opts));
+ ASSERT_EQ(bbto->block_align, true);
+ ASSERT_EQ(bbto->block_size, 8192);
+}
+
+TEST_F(OptionsTest, MutableCFOptions) {
+ ConfigOptions config_options;
+ ColumnFamilyOptions cf_opts;
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, cf_opts,
+ "paranoid_file_checks=true; block_based_table_factory.block_align=false; "
+ "block_based_table_factory.block_size=8192;",
+ &cf_opts));
+ ASSERT_TRUE(cf_opts.paranoid_file_checks);
+ ASSERT_NE(cf_opts.table_factory.get(), nullptr);
+ const auto bbto = cf_opts.table_factory->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(bbto, nullptr);
+ ASSERT_EQ(bbto->block_size, 8192);
+ ASSERT_EQ(bbto->block_align, false);
+ std::unordered_map<std::string, std::string> unused_opts;
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts, {{"paranoid_file_checks", "false"}}, &cf_opts));
+ ASSERT_EQ(cf_opts.paranoid_file_checks, false);
+
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts,
+ {{"block_based_table_factory.block_size", "16384"}}, &cf_opts));
+ ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions<BlockBasedTableOptions>());
+ ASSERT_EQ(bbto->block_size, 16384);
+
+ config_options.mutable_options_only = true;
+ // Force consistency checks is not mutable
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts, {{"force_consistency_checks", "true"}},
+ &cf_opts));
+
+ // Attempt to change the table. It is not mutable, so this should fail and
+ // leave the original intact
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts, {{"table_factory", "PlainTable"}}, &cf_opts));
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts, {{"table_factory.id", "PlainTable"}}, &cf_opts));
+ ASSERT_NE(cf_opts.table_factory.get(), nullptr);
+ ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions<BlockBasedTableOptions>());
+
+ // Change the block size. Should update the value in the current table
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts,
+ {{"block_based_table_factory.block_size", "8192"}}, &cf_opts));
+ ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions<BlockBasedTableOptions>());
+ ASSERT_EQ(bbto->block_size, 8192);
+
+ // Attempt to turn off block cache fails, as this option is not mutable
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts,
+ {{"block_based_table_factory.no_block_cache", "true"}}, &cf_opts));
+ ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions<BlockBasedTableOptions>());
+
+ // Attempt to change the block size via a config string/map. Should update
+ // the current value
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts,
+ {{"block_based_table_factory", "{block_size=32768}"}}, &cf_opts));
+ ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions<BlockBasedTableOptions>());
+ ASSERT_EQ(bbto->block_size, 32768);
+
+ // Attempt to change the block size and no cache through the map. Should
+ // fail, leaving the old values intact
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ config_options, cf_opts,
+ {{"block_based_table_factory",
+ "{block_size=16384; no_block_cache=true}"}},
+ &cf_opts));
+ ASSERT_EQ(bbto, cf_opts.table_factory->GetOptions<BlockBasedTableOptions>());
+ ASSERT_EQ(bbto->block_size, 32768);
+}
+
+#endif // !ROCKSDB_LITE
+
+Status StringToMap(
+ const std::string& opts_str,
+ std::unordered_map<std::string, std::string>* opts_map);
+
+#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
+TEST_F(OptionsTest, StringToMapTest) {
+ std::unordered_map<std::string, std::string> opts_map;
+ // Regular options
+ ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "v2");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ // Value with '='
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "=v1");
+ ASSERT_EQ(opts_map["k2"], "v2=");
+ // Overwrriten option
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v2");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ // Empty value
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
+ ASSERT_EQ(opts_map["k4"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
+ ASSERT_EQ(opts_map["k4"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
+ ASSERT_EQ(opts_map["k3"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
+ ASSERT_EQ(opts_map["k2"], "");
+ ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
+ ASSERT_EQ(opts_map["k3"], "");
+ // Regular nested options
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2");
+ ASSERT_EQ(opts_map["k3"], "v3");
+ // Multi-level nested options
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};"
+ "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4",
+ &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}");
+ ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}");
+ ASSERT_EQ(opts_map["k4"], "v4");
+ // Garbage inside curly braces
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4",
+ &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "dfad=");
+ ASSERT_EQ(opts_map["k3"], "=");
+ ASSERT_EQ(opts_map["k4"], "v4");
+ // Empty nested options
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "");
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}");
+ // With random spaces
+ opts_map.clear();
+ ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; "
+ "k3={ { } }; k4= v4 ",
+ &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}");
+ ASSERT_EQ(opts_map["k3"], "{ }");
+ ASSERT_EQ(opts_map["k4"], "v4");
+
+ // Empty key
+ ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map));
+ ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map));
+ // Mismatch curly braces
+ ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map));
+ // However this is valid!
+ opts_map.clear();
+ ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map));
+ ASSERT_EQ(opts_map["k1"], "v1");
+ ASSERT_EQ(opts_map["k2"], "}");
+ ASSERT_EQ(opts_map["k3"], "v3");
+
+ // Invalid chars after closing curly brace
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map));
+ ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map));
+}
+#endif // ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
+TEST_F(OptionsTest, StringToMapRandomTest) {
+ std::unordered_map<std::string, std::string> opts_map;
+ // Make sure segfault is not hit by semi-random strings
+
+ std::vector<std::string> bases = {
+ "a={aa={};tt={xxx={}}};c=defff",
+ "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}",
+ "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"};
+
+ for (std::string base : bases) {
+ for (int rand_seed = 301; rand_seed < 401; rand_seed++) {
+ Random rnd(rand_seed);
+ for (int attempt = 0; attempt < 10; attempt++) {
+ std::string str = base;
+ // Replace random position to space
+ size_t pos = static_cast<size_t>(
+ rnd.Uniform(static_cast<int>(base.size())));
+ str[pos] = ' ';
+ Status s = StringToMap(str, &opts_map);
+ ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+ opts_map.clear();
+ }
+ }
+ }
+
+ // Random Construct a string
+ std::vector<char> chars = {'{', '}', ' ', '=', ';', 'c'};
+ for (int rand_seed = 301; rand_seed < 1301; rand_seed++) {
+ Random rnd(rand_seed);
+ int len = rnd.Uniform(30);
+ std::string str = "";
+ for (int attempt = 0; attempt < len; attempt++) {
+ // Add a random character
+ size_t pos = static_cast<size_t>(
+ rnd.Uniform(static_cast<int>(chars.size())));
+ str.append(1, chars[pos]);
+ }
+ Status s = StringToMap(str, &opts_map);
+ ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+ s = StringToMap("name=" + str, &opts_map);
+ ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
+ opts_map.clear();
+ }
+}
+
+TEST_F(OptionsTest, GetStringFromCompressionType) {
+ std::string res;
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression));
+ ASSERT_EQ(res, "kNoCompression");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression));
+ ASSERT_EQ(res, "kSnappyCompression");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption));
+ ASSERT_EQ(res, "kDisableCompressionOption");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression));
+ ASSERT_EQ(res, "kLZ4Compression");
+
+ ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression));
+ ASSERT_EQ(res, "kZlibCompression");
+
+ ASSERT_NOK(
+ GetStringFromCompressionType(&res, static_cast<CompressionType>(-10)));
+}
+
+TEST_F(OptionsTest, OnlyMutableDBOptions) {
+ std::string opt_str;
+ Random rnd(302);
+ ConfigOptions cfg_opts;
+ DBOptions db_opts;
+ DBOptions mdb_opts;
+ std::unordered_set<std::string> m_names;
+ std::unordered_set<std::string> a_names;
+
+ test::RandomInitDBOptions(&db_opts, &rnd);
+ auto db_config = DBOptionsAsConfigurable(db_opts);
+
+ // Get all of the DB Option names (mutable or not)
+ ASSERT_OK(db_config->GetOptionNames(cfg_opts, &a_names));
+
+ // Get only the mutable options from db_opts and set those in mdb_opts
+ cfg_opts.mutable_options_only = true;
+
+ // Get only the Mutable DB Option names
+ ASSERT_OK(db_config->GetOptionNames(cfg_opts, &m_names));
+ ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opt_str));
+ ASSERT_OK(GetDBOptionsFromString(cfg_opts, mdb_opts, opt_str, &mdb_opts));
+ std::string mismatch;
+ // Comparing only the mutable options, the two are equivalent
+ auto mdb_config = DBOptionsAsConfigurable(mdb_opts);
+ ASSERT_TRUE(mdb_config->AreEquivalent(cfg_opts, db_config.get(), &mismatch));
+ ASSERT_TRUE(db_config->AreEquivalent(cfg_opts, mdb_config.get(), &mismatch));
+
+ ASSERT_GT(a_names.size(), m_names.size());
+ for (const auto& n : m_names) {
+ std::string m, d;
+ ASSERT_OK(mdb_config->GetOption(cfg_opts, n, &m));
+ ASSERT_OK(db_config->GetOption(cfg_opts, n, &d));
+ ASSERT_EQ(m, d);
+ }
+
+ cfg_opts.mutable_options_only = false;
+ // Comparing all of the options, the two are not equivalent
+ ASSERT_FALSE(mdb_config->AreEquivalent(cfg_opts, db_config.get(), &mismatch));
+ ASSERT_FALSE(db_config->AreEquivalent(cfg_opts, mdb_config.get(), &mismatch));
+
+ // Make sure there are only mutable options being configured
+ ASSERT_OK(GetDBOptionsFromString(cfg_opts, DBOptions(), opt_str, &db_opts));
+}
+
+TEST_F(OptionsTest, OnlyMutableCFOptions) {
+ std::string opt_str;
+ Random rnd(302);
+ ConfigOptions cfg_opts;
+ DBOptions db_opts;
+ ColumnFamilyOptions mcf_opts;
+ ColumnFamilyOptions cf_opts;
+ std::unordered_set<std::string> m_names;
+ std::unordered_set<std::string> a_names;
+
+ test::RandomInitCFOptions(&cf_opts, db_opts, &rnd);
+ cf_opts.comparator = ReverseBytewiseComparator();
+ auto cf_config = CFOptionsAsConfigurable(cf_opts);
+
+ // Get all of the CF Option names (mutable or not)
+ ASSERT_OK(cf_config->GetOptionNames(cfg_opts, &a_names));
+
+ // Get only the mutable options from cf_opts and set those in mcf_opts
+ cfg_opts.mutable_options_only = true;
+ // Get only the Mutable CF Option names
+ ASSERT_OK(cf_config->GetOptionNames(cfg_opts, &m_names));
+ ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, cf_opts, &opt_str));
+ ASSERT_OK(
+ GetColumnFamilyOptionsFromString(cfg_opts, mcf_opts, opt_str, &mcf_opts));
+ std::string mismatch;
+
+ auto mcf_config = CFOptionsAsConfigurable(mcf_opts);
+ // Comparing only the mutable options, the two are equivalent
+ ASSERT_TRUE(mcf_config->AreEquivalent(cfg_opts, cf_config.get(), &mismatch));
+ ASSERT_TRUE(cf_config->AreEquivalent(cfg_opts, mcf_config.get(), &mismatch));
+
+ ASSERT_GT(a_names.size(), m_names.size());
+ for (const auto& n : m_names) {
+ std::string m, d;
+ ASSERT_OK(mcf_config->GetOption(cfg_opts, n, &m));
+ ASSERT_OK(cf_config->GetOption(cfg_opts, n, &d));
+ ASSERT_EQ(m, d);
+ }
+
+ cfg_opts.mutable_options_only = false;
+ // Comparing all of the options, the two are not equivalent
+ ASSERT_FALSE(mcf_config->AreEquivalent(cfg_opts, cf_config.get(), &mismatch));
+ ASSERT_FALSE(cf_config->AreEquivalent(cfg_opts, mcf_config.get(), &mismatch));
+ delete cf_opts.compaction_filter;
+
+ // Make sure the options string contains only mutable options
+ ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, ColumnFamilyOptions(),
+ opt_str, &cf_opts));
+ delete cf_opts.compaction_filter;
+}
+
+TEST_F(OptionsTest, SstPartitionerTest) {
+ ConfigOptions cfg_opts;
+ ColumnFamilyOptions cf_opts, new_opt;
+ std::string opts_str, mismatch;
+
+ ASSERT_OK(SstPartitionerFactory::CreateFromString(
+ cfg_opts, SstPartitionerFixedPrefixFactory::kClassName(),
+ &cf_opts.sst_partitioner_factory));
+ ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr);
+ ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(),
+ SstPartitionerFixedPrefixFactory::kClassName());
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ cfg_opts, ColumnFamilyOptions(),
+ std::string("sst_partitioner_factory={id=") +
+ SstPartitionerFixedPrefixFactory::kClassName() + "; unknown=10;}",
+ &cf_opts));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ cfg_opts, ColumnFamilyOptions(),
+ std::string("sst_partitioner_factory={id=") +
+ SstPartitionerFixedPrefixFactory::kClassName() + "; length=10;}",
+ &cf_opts));
+ ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr);
+ ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(),
+ SstPartitionerFixedPrefixFactory::kClassName());
+ ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, cf_opts, &opts_str));
+ ASSERT_OK(
+ GetColumnFamilyOptionsFromString(cfg_opts, cf_opts, opts_str, &new_opt));
+ ASSERT_NE(new_opt.sst_partitioner_factory, nullptr);
+ ASSERT_STREQ(new_opt.sst_partitioner_factory->Name(),
+ SstPartitionerFixedPrefixFactory::kClassName());
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, cf_opts, new_opt));
+ ASSERT_TRUE(cf_opts.sst_partitioner_factory->AreEquivalent(
+ cfg_opts, new_opt.sst_partitioner_factory.get(), &mismatch));
+}
+
+TEST_F(OptionsTest, FileChecksumGenFactoryTest) {
+ ConfigOptions cfg_opts;
+ DBOptions db_opts, new_opt;
+ std::string opts_str, mismatch;
+ auto factory = GetFileChecksumGenCrc32cFactory();
+
+ cfg_opts.ignore_unsupported_options = false;
+
+ ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str));
+ ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt));
+
+ ASSERT_NE(factory, nullptr);
+ ASSERT_OK(FileChecksumGenFactory::CreateFromString(
+ cfg_opts, factory->Name(), &db_opts.file_checksum_gen_factory));
+ ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr);
+ ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name());
+ ASSERT_NOK(GetDBOptionsFromString(
+ cfg_opts, DBOptions(), "file_checksum_gen_factory=unknown", &db_opts));
+ ASSERT_OK(GetDBOptionsFromString(
+ cfg_opts, DBOptions(),
+ std::string("file_checksum_gen_factory=") + factory->Name(), &db_opts));
+ ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr);
+ ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name());
+
+ ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str));
+ ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt));
+ ASSERT_NE(new_opt.file_checksum_gen_factory, nullptr);
+ ASSERT_STREQ(new_opt.file_checksum_gen_factory->Name(), factory->Name());
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(cfg_opts, db_opts, new_opt));
+ ASSERT_TRUE(factory->AreEquivalent(
+ cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch));
+ ASSERT_TRUE(db_opts.file_checksum_gen_factory->AreEquivalent(
+ cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch));
+}
+
+class TestTablePropertiesCollectorFactory
+ : public TablePropertiesCollectorFactory {
+ private:
+ std::string id_;
+
+ public:
+ explicit TestTablePropertiesCollectorFactory(const std::string& id)
+ : id_(id) {}
+ TablePropertiesCollector* CreateTablePropertiesCollector(
+ TablePropertiesCollectorFactory::Context /*context*/) override {
+ return nullptr;
+ }
+ static const char* kClassName() { return "TestCollector"; }
+ const char* Name() const override { return kClassName(); }
+ std::string GetId() const override {
+ return std::string(kClassName()) + ":" + id_;
+ }
+};
+
+TEST_F(OptionsTest, OptionTablePropertiesTest) {
+ ConfigOptions cfg_opts;
+ ColumnFamilyOptions orig, copy;
+ orig.table_properties_collector_factories.push_back(
+ std::make_shared<TestTablePropertiesCollectorFactory>("1"));
+ orig.table_properties_collector_factories.push_back(
+ std::make_shared<TestTablePropertiesCollectorFactory>("2"));
+
+ // Push two TablePropertiesCollectorFactories then create a new
+ // ColumnFamilyOptions based on those settings. The copy should
+ // have no properties but still match the original
+ std::string opts_str;
+ ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, orig, &opts_str));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, &copy));
+ ASSERT_EQ(copy.table_properties_collector_factories.size(), 0);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy));
+
+ // Now register a TablePropertiesCollectorFactory
+ // Repeat the experiment. The copy should have the same
+ // properties as the original
+ cfg_opts.registry->AddLibrary("collector")
+ ->AddFactory<TablePropertiesCollectorFactory>(
+ ObjectLibrary::PatternEntry(
+ TestTablePropertiesCollectorFactory::kClassName(), false)
+ .AddSeparator(":"),
+ [](const std::string& name,
+ std::unique_ptr<TablePropertiesCollectorFactory>* guard,
+ std::string* /* errmsg */) {
+ std::string id = name.substr(
+ strlen(TestTablePropertiesCollectorFactory::kClassName()) + 1);
+ guard->reset(new TestTablePropertiesCollectorFactory(id));
+ return guard->get();
+ });
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, &copy));
+ ASSERT_EQ(copy.table_properties_collector_factories.size(), 2);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy));
+}
+#endif // !ROCKSDB_LITE
+
+TEST_F(OptionsTest, ConvertOptionsTest) {
+ LevelDBOptions leveldb_opt;
+ Options converted_opt = ConvertOptions(leveldb_opt);
+
+ ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing);
+ ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists);
+ ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks);
+ ASSERT_EQ(converted_opt.env, leveldb_opt.env);
+ ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log);
+ ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size);
+ ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files);
+ ASSERT_EQ(converted_opt.compression, leveldb_opt.compression);
+
+ std::shared_ptr<TableFactory> table_factory = converted_opt.table_factory;
+ const auto table_opt = table_factory->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(table_opt, nullptr);
+
+ ASSERT_EQ(table_opt->block_cache->GetCapacity(), 8UL << 20);
+ ASSERT_EQ(table_opt->block_size, leveldb_opt.block_size);
+ ASSERT_EQ(table_opt->block_restart_interval,
+ leveldb_opt.block_restart_interval);
+ ASSERT_EQ(table_opt->filter_policy.get(), leveldb_opt.filter_policy);
+}
+#ifndef ROCKSDB_LITE
+class TestEventListener : public EventListener {
+ private:
+ std::string id_;
+
+ public:
+ explicit TestEventListener(const std::string& id) : id_("Test" + id) {}
+ const char* Name() const override { return id_.c_str(); }
+};
+
+static std::unordered_map<std::string, OptionTypeInfo>
+ test_listener_option_info = {
+ {"s",
+ {0, OptionType::kString, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+
+};
+
+class TestConfigEventListener : public TestEventListener {
+ private:
+ std::string s_;
+
+ public:
+ explicit TestConfigEventListener(const std::string& id)
+ : TestEventListener("Config" + id) {
+ s_ = id;
+ RegisterOptions("Test", &s_, &test_listener_option_info);
+ }
+};
+
+static int RegisterTestEventListener(ObjectLibrary& library,
+ const std::string& arg) {
+ library.AddFactory<EventListener>(
+ "Test" + arg,
+ [](const std::string& name, std::unique_ptr<EventListener>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new TestEventListener(name.substr(4)));
+ return guard->get();
+ });
+ library.AddFactory<EventListener>(
+ "TestConfig" + arg,
+ [](const std::string& name, std::unique_ptr<EventListener>* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new TestConfigEventListener(name.substr(10)));
+ return guard->get();
+ });
+ return 1;
+}
+TEST_F(OptionsTest, OptionsListenerTest) {
+ DBOptions orig, copy;
+ orig.listeners.push_back(std::make_shared<TestEventListener>("1"));
+ orig.listeners.push_back(std::make_shared<TestEventListener>("2"));
+ orig.listeners.push_back(std::make_shared<TestEventListener>(""));
+ orig.listeners.push_back(std::make_shared<TestConfigEventListener>("1"));
+ orig.listeners.push_back(std::make_shared<TestConfigEventListener>("2"));
+ orig.listeners.push_back(std::make_shared<TestConfigEventListener>(""));
+ ConfigOptions config_opts(orig);
+ config_opts.registry->AddLibrary("listener", RegisterTestEventListener, "1");
+ std::string opts_str;
+ ASSERT_OK(GetStringFromDBOptions(config_opts, orig, &opts_str));
+ ASSERT_OK(GetDBOptionsFromString(config_opts, orig, opts_str, &copy));
+ ASSERT_OK(GetStringFromDBOptions(config_opts, copy, &opts_str));
+ ASSERT_EQ(
+ copy.listeners.size(),
+ 2); // The Test{Config}1 Listeners could be loaded but not the others
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_opts, orig, copy));
+}
+#endif // ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE
+const static std::string kCustomEnvName = "Custom";
+const static std::string kCustomEnvProp = "env=" + kCustomEnvName;
+
+static int RegisterCustomEnv(ObjectLibrary& library, const std::string& arg) {
+ library.AddFactory<Env>(
+ arg, [](const std::string& /*name*/, std::unique_ptr<Env>* /*env_guard*/,
+ std::string* /* errmsg */) {
+ static CustomEnv env(Env::Default());
+ return &env;
+ });
+ return 1;
+}
+
+// This test suite tests the old APIs into the Configure options methods.
+// Once those APIs are officially deprecated, this test suite can be deleted.
+class OptionsOldApiTest : public testing::Test {};
+
+TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
+ std::unordered_map<std::string, std::string> cf_options_map = {
+ {"write_buffer_size", "1"},
+ {"max_write_buffer_number", "2"},
+ {"min_write_buffer_number_to_merge", "3"},
+ {"max_write_buffer_number_to_maintain", "99"},
+ {"max_write_buffer_size_to_maintain", "-99999"},
+ {"compression", "kSnappyCompression"},
+ {"compression_per_level",
+ "kNoCompression:"
+ "kSnappyCompression:"
+ "kZlibCompression:"
+ "kBZip2Compression:"
+ "kLZ4Compression:"
+ "kLZ4HCCompression:"
+ "kXpressCompression:"
+ "kZSTD:"
+ "kZSTDNotFinalCompression"},
+ {"bottommost_compression", "kLZ4Compression"},
+ {"bottommost_compression_opts", "5:6:7:8:9:true"},
+ {"compression_opts", "4:5:6:7:8:9:true:10:false"},
+ {"num_levels", "8"},
+ {"level0_file_num_compaction_trigger", "8"},
+ {"level0_slowdown_writes_trigger", "9"},
+ {"level0_stop_writes_trigger", "10"},
+ {"target_file_size_base", "12"},
+ {"target_file_size_multiplier", "13"},
+ {"max_bytes_for_level_base", "14"},
+ {"level_compaction_dynamic_level_bytes", "true"},
+ {"level_compaction_dynamic_file_size", "true"},
+ {"max_bytes_for_level_multiplier", "15.0"},
+ {"max_bytes_for_level_multiplier_additional", "16:17:18"},
+ {"max_compaction_bytes", "21"},
+ {"soft_rate_limit", "1.1"},
+ {"hard_rate_limit", "2.1"},
+ {"rate_limit_delay_max_milliseconds", "100"},
+ {"hard_pending_compaction_bytes_limit", "211"},
+ {"arena_block_size", "22"},
+ {"disable_auto_compactions", "true"},
+ {"compaction_style", "kCompactionStyleLevel"},
+ {"compaction_pri", "kOldestSmallestSeqFirst"},
+ {"verify_checksums_in_compaction", "false"},
+ {"compaction_options_fifo", "23"},
+ {"max_sequential_skip_in_iterations", "24"},
+ {"inplace_update_support", "true"},
+ {"report_bg_io_stats", "true"},
+ {"compaction_measure_io_stats", "false"},
+ {"purge_redundant_kvs_while_flush", "false"},
+ {"inplace_update_num_locks", "25"},
+ {"memtable_prefix_bloom_size_ratio", "0.26"},
+ {"memtable_whole_key_filtering", "true"},
+ {"memtable_huge_page_size", "28"},
+ {"bloom_locality", "29"},
+ {"max_successive_merges", "30"},
+ {"min_partial_merge_operands", "31"},
+ {"prefix_extractor", "fixed:31"},
+ {"experimental_mempurge_threshold", "0.003"},
+ {"optimize_filters_for_hits", "true"},
+ {"enable_blob_files", "true"},
+ {"min_blob_size", "1K"},
+ {"blob_file_size", "1G"},
+ {"blob_compression_type", "kZSTD"},
+ {"enable_blob_garbage_collection", "true"},
+ {"blob_garbage_collection_age_cutoff", "0.5"},
+ {"blob_garbage_collection_force_threshold", "0.75"},
+ {"blob_compaction_readahead_size", "256K"},
+ {"blob_file_starting_level", "1"},
+ {"prepopulate_blob_cache", "kDisable"},
+ {"last_level_temperature", "kWarm"},
+ };
+
+ std::unordered_map<std::string, std::string> db_options_map = {
+ {"create_if_missing", "false"},
+ {"create_missing_column_families", "true"},
+ {"error_if_exists", "false"},
+ {"paranoid_checks", "true"},
+ {"track_and_verify_wals_in_manifest", "true"},
+ {"verify_sst_unique_id_in_manifest", "true"},
+ {"max_open_files", "32"},
+ {"max_total_wal_size", "33"},
+ {"use_fsync", "true"},
+ {"db_log_dir", "/db_log_dir"},
+ {"wal_dir", "/wal_dir"},
+ {"delete_obsolete_files_period_micros", "34"},
+ {"max_background_compactions", "35"},
+ {"max_background_flushes", "36"},
+ {"max_log_file_size", "37"},
+ {"log_file_time_to_roll", "38"},
+ {"keep_log_file_num", "39"},
+ {"recycle_log_file_num", "5"},
+ {"max_manifest_file_size", "40"},
+ {"table_cache_numshardbits", "41"},
+ {"WAL_ttl_seconds", "43"},
+ {"WAL_size_limit_MB", "44"},
+ {"manifest_preallocation_size", "45"},
+ {"allow_mmap_reads", "true"},
+ {"allow_mmap_writes", "false"},
+ {"use_direct_reads", "false"},
+ {"use_direct_io_for_flush_and_compaction", "false"},
+ {"is_fd_close_on_exec", "true"},
+ {"skip_log_error_on_recovery", "false"},
+ {"stats_dump_period_sec", "46"},
+ {"stats_persist_period_sec", "57"},
+ {"persist_stats_to_disk", "false"},
+ {"stats_history_buffer_size", "69"},
+ {"advise_random_on_open", "true"},
+ {"use_adaptive_mutex", "false"},
+ {"compaction_readahead_size", "100"},
+ {"random_access_max_buffer_size", "3145728"},
+ {"writable_file_max_buffer_size", "314159"},
+ {"bytes_per_sync", "47"},
+ {"wal_bytes_per_sync", "48"},
+ {"strict_bytes_per_sync", "true"},
+ {"preserve_deletes", "false"},
+ };
+
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
+ ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999);
+ ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
+ ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
+ ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
+ ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
+ ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 9u);
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_buffer_bytes, 10u);
+ ASSERT_EQ(new_cf_opt.compression_opts.use_zstd_dict_trainer, false);
+ ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads,
+ CompressionOptions().parallel_threads);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_buffer_bytes,
+ CompressionOptions().max_dict_buffer_bytes);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.use_zstd_dict_trainer,
+ CompressionOptions().use_zstd_dict_trainer);
+ ASSERT_EQ(new_cf_opt.num_levels, 8);
+ ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8);
+ ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9);
+ ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10);
+ ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12));
+ ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
+ ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
+ ASSERT_EQ(new_cf_opt.level_compaction_dynamic_file_size, true);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18);
+ ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21);
+ ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
+ ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
+ ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
+ ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
+ ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
+ static_cast<uint64_t>(23));
+ ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
+ static_cast<uint64_t>(24));
+ ASSERT_EQ(new_cf_opt.inplace_update_support, true);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
+ ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
+ ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true);
+ ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
+ ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
+ ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
+ ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
+ ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.FixedPrefix.31");
+ ASSERT_EQ(new_cf_opt.experimental_mempurge_threshold, 0.003);
+ ASSERT_EQ(new_cf_opt.enable_blob_files, true);
+ ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
+ ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
+ ASSERT_EQ(new_cf_opt.blob_compression_type, kZSTD);
+ ASSERT_EQ(new_cf_opt.enable_blob_garbage_collection, true);
+ ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
+ ASSERT_EQ(new_cf_opt.blob_garbage_collection_force_threshold, 0.75);
+ ASSERT_EQ(new_cf_opt.blob_compaction_readahead_size, 262144);
+ ASSERT_EQ(new_cf_opt.blob_file_starting_level, 1);
+ ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable);
+ ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm);
+ ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
+
+ cf_options_map["write_buffer_size"] = "hello";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ConfigOptions exact, loose;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ cf_options_map["write_buffer_size"] = "1";
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+
+ cf_options_map["unknown_option"] = "1";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
+ &new_cf_opt,
+ false, /* input_strings_escaped */
+ true /* ignore_unknown_options */));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ loose, base_cf_opt, new_cf_opt, nullptr /* new_opt_map */));
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ exact /* default for VerifyCFOptions */, base_cf_opt, new_cf_opt, nullptr));
+
+ DBOptions base_db_opt;
+ DBOptions new_db_opt;
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.create_missing_column_families, true);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.paranoid_checks, true);
+ ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
+ ASSERT_EQ(new_db_opt.max_open_files, 32);
+ ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
+ ASSERT_EQ(new_db_opt.use_fsync, true);
+ ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir");
+ ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir");
+ ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros,
+ static_cast<uint64_t>(34));
+ ASSERT_EQ(new_db_opt.max_background_compactions, 35);
+ ASSERT_EQ(new_db_opt.max_background_flushes, 36);
+ ASSERT_EQ(new_db_opt.max_log_file_size, 37U);
+ ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U);
+ ASSERT_EQ(new_db_opt.keep_log_file_num, 39U);
+ ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U);
+ ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40));
+ ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41);
+ ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43));
+ ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44));
+ ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U);
+ ASSERT_EQ(new_db_opt.allow_mmap_reads, true);
+ ASSERT_EQ(new_db_opt.allow_mmap_writes, false);
+ ASSERT_EQ(new_db_opt.use_direct_reads, false);
+ ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false);
+ ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true);
+ ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U);
+ ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U);
+ ASSERT_EQ(new_db_opt.persist_stats_to_disk, false);
+ ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U);
+ ASSERT_EQ(new_db_opt.advise_random_on_open, true);
+ ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
+ ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
+ ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
+ ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
+ ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
+ ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
+ ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true);
+
+ db_options_map["max_open_files"] = "hello";
+ ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
+
+ // unknow options should fail parsing without ignore_unknown_options = true
+ db_options_map["unknown_db_option"] = "1";
+ ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt,
+ false, /* input_strings_escaped */
+ true /* ignore_unknown_options */));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
+ ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+}
+
+TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) {
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ base_cf_opt.table_factory.reset();
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=5", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
+ ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=6;", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ " write_buffer_size = 7 ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ " write_buffer_size = 8 ; ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=11; max_write_buffer_number = 12 ;",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
+ // Wrong name "max_write_buffer_number_"
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number_=14;",
+ &new_cf_opt));
+ ConfigOptions exact;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Comparator from object registry
+ std::string kCompName = "reverse_comp";
+ ObjectLibrary::Default()->AddFactory<const Comparator>(
+ kCompName,
+ [](const std::string& /*name*/,
+ std::unique_ptr<const Comparator>* /*guard*/,
+ std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator());
+
+ // MergeOperator from object registry
+ std::unique_ptr<BytesXOROperator> bxo(new BytesXOROperator());
+ std::string kMoName = bxo->Name();
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "merge_operator=" + kMoName + ";", &new_cf_opt));
+ ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name()));
+
+ // Wrong key/value pair
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Error Paring value
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Missing option name
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13; =100;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ const uint64_t kilo = 1024UL;
+ const uint64_t mega = 1024 * kilo;
+ const uint64_t giga = 1024 * mega;
+ const uint64_t tera = 1024 * giga;
+
+ // Units (k)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo);
+ // Units (m)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "max_write_buffer_number=16m;inplace_update_num_locks=17M",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega);
+ // Units (g)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "write_buffer_size=18g;prefix_extractor=capped:8;"
+ "arena_block_size=19G",
+ &new_cf_opt));
+
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
+ ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.CappedPrefix.8");
+
+ // Units (t)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
+
+ // Nested block based table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Last one
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;}",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Mismatch curly braces
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={{{block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Unexpected chars after closing curly brace
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa;"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Invalid block based table option
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={xx_block_size=4;}",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=true",
+ &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=false",
+ &new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=junk",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Nested plain table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+
+ // memtable factory
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "memtable=skip_list:10;arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
+ ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
+
+ // blob cache
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "blob_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};",
+ &new_cf_opt));
+ ASSERT_NE(new_cf_opt.blob_cache, nullptr);
+ ASSERT_EQ(new_cf_opt.blob_cache->GetCapacity(), 1024UL * 1024UL);
+ ASSERT_EQ(static_cast<ShardedCacheBase*>(new_cf_opt.blob_cache.get())
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_cf_opt.blob_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(static_cast<LRUCache*>(new_cf_opt.blob_cache.get())
+ ->GetHighPriPoolRatio(),
+ 0.5);
+}
+
+TEST_F(OptionsTest, SliceTransformCreateFromString) {
+ std::shared_ptr<const SliceTransform> transform = nullptr;
+ ConfigOptions config_options;
+ config_options.ignore_unsupported_options = false;
+ config_options.ignore_unknown_options = false;
+
+ ASSERT_OK(
+ SliceTransform::CreateFromString(config_options, "fixed:31", &transform));
+ ASSERT_NE(transform, nullptr);
+ ASSERT_FALSE(transform->IsInstanceOf("capped"));
+ ASSERT_TRUE(transform->IsInstanceOf("fixed"));
+ ASSERT_TRUE(transform->IsInstanceOf("rocksdb.FixedPrefix"));
+ ASSERT_EQ(transform->GetId(), "rocksdb.FixedPrefix.31");
+ ASSERT_OK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.FixedPrefix.42", &transform));
+ ASSERT_NE(transform, nullptr);
+ ASSERT_EQ(transform->GetId(), "rocksdb.FixedPrefix.42");
+
+ ASSERT_OK(SliceTransform::CreateFromString(config_options, "capped:16",
+ &transform));
+ ASSERT_NE(transform, nullptr);
+ ASSERT_FALSE(transform->IsInstanceOf("fixed"));
+ ASSERT_TRUE(transform->IsInstanceOf("capped"));
+ ASSERT_TRUE(transform->IsInstanceOf("rocksdb.CappedPrefix"));
+ ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.16");
+ ASSERT_OK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.CappedPrefix.42", &transform));
+ ASSERT_NE(transform, nullptr);
+ ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.42");
+
+ ASSERT_OK(SliceTransform::CreateFromString(config_options, "rocksdb.Noop",
+ &transform));
+ ASSERT_NE(transform, nullptr);
+
+ ASSERT_NOK(SliceTransform::CreateFromString(config_options,
+ "fixed:21:invalid", &transform));
+ ASSERT_NOK(SliceTransform::CreateFromString(config_options,
+ "capped:21:invalid", &transform));
+ ASSERT_NOK(
+ SliceTransform::CreateFromString(config_options, "fixed", &transform));
+ ASSERT_NOK(
+ SliceTransform::CreateFromString(config_options, "capped", &transform));
+ ASSERT_NOK(
+ SliceTransform::CreateFromString(config_options, "fixed:", &transform));
+ ASSERT_NOK(
+ SliceTransform::CreateFromString(config_options, "capped:", &transform));
+ ASSERT_NOK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.FixedPrefix:42", &transform));
+ ASSERT_NOK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.CappedPrefix:42", &transform));
+ ASSERT_NOK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.FixedPrefix", &transform));
+ ASSERT_NOK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.CappedPrefix", &transform));
+ ASSERT_NOK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.FixedPrefix.", &transform));
+ ASSERT_NOK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.CappedPrefix.", &transform));
+ ASSERT_NOK(
+ SliceTransform::CreateFromString(config_options, "invalid", &transform));
+
+#ifndef ROCKSDB_LITE
+ ASSERT_OK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.CappedPrefix.11", &transform));
+ ASSERT_NE(transform, nullptr);
+ ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.11");
+ ASSERT_TRUE(transform->IsInstanceOf("capped"));
+ ASSERT_TRUE(transform->IsInstanceOf("capped:11"));
+ ASSERT_TRUE(transform->IsInstanceOf("rocksdb.CappedPrefix"));
+ ASSERT_TRUE(transform->IsInstanceOf("rocksdb.CappedPrefix.11"));
+ ASSERT_FALSE(transform->IsInstanceOf("fixed"));
+ ASSERT_FALSE(transform->IsInstanceOf("fixed:11"));
+ ASSERT_FALSE(transform->IsInstanceOf("rocksdb.FixedPrefix"));
+ ASSERT_FALSE(transform->IsInstanceOf("rocksdb.FixedPrefix.11"));
+
+ ASSERT_OK(SliceTransform::CreateFromString(
+ config_options, "rocksdb.FixedPrefix.11", &transform));
+ ASSERT_TRUE(transform->IsInstanceOf("fixed"));
+ ASSERT_TRUE(transform->IsInstanceOf("fixed:11"));
+ ASSERT_TRUE(transform->IsInstanceOf("rocksdb.FixedPrefix"));
+ ASSERT_TRUE(transform->IsInstanceOf("rocksdb.FixedPrefix.11"));
+ ASSERT_FALSE(transform->IsInstanceOf("capped"));
+ ASSERT_FALSE(transform->IsInstanceOf("capped:11"));
+ ASSERT_FALSE(transform->IsInstanceOf("rocksdb.CappedPrefix"));
+ ASSERT_FALSE(transform->IsInstanceOf("rocksdb.CappedPrefix.11"));
+#endif // ROCKSDB_LITE
+}
+
+TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) {
+ BlockBasedTableOptions table_opt;
+ BlockBasedTableOptions new_opt;
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kHashSearch;"
+ "checksum=kxxHash;no_block_cache=1;"
+ "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
+ "block_size_deviation=8;block_restart_interval=4;"
+ "format_version=5;whole_key_filtering=1;"
+ "filter_policy=bloomfilter:4.567:false;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
+ ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
+ ASSERT_TRUE(new_opt.no_block_cache);
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL);
+ ASSERT_EQ(new_opt.block_size, 1024UL);
+ ASSERT_EQ(new_opt.block_size_deviation, 8);
+ ASSERT_EQ(new_opt.block_restart_interval, 4);
+ ASSERT_EQ(new_opt.format_version, 5U);
+ ASSERT_EQ(new_opt.whole_key_filtering, true);
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ const BloomFilterPolicy* bfp =
+ dynamic_cast<const BloomFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(bfp->GetMillibitsPerKey(), 4567);
+ EXPECT_EQ(bfp->GetWholeBitsPerKey(), 5);
+
+ // unknown option
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
+ "bad_option=1",
+ &new_opt));
+ ASSERT_EQ(static_cast<bool>(table_opt.cache_index_and_filter_blocks),
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized index type
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized checksum type
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;checksum=kxxHashXX",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized filter policy name
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;"
+ "filter_policy=bloomfilterxx:4:true",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
+
+ // Used to be rejected, now accepted
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ table_opt, "filter_policy=bloomfilter:4", &new_opt));
+ bfp = dynamic_cast<const BloomFilterPolicy*>(new_opt.filter_policy.get());
+ EXPECT_EQ(bfp->GetMillibitsPerKey(), 4000);
+ EXPECT_EQ(bfp->GetWholeBitsPerKey(), 4);
+
+ // Check block cache options are overwritten when specified
+ // in new format as a struct.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set only block cache capacity. Check other values are
+ // reset to default values.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=2M};"
+ "block_cache_compressed={capacity=2M}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(
+ std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ GetDefaultCacheShardBits(new_opt.block_cache_compressed->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ table_opt,
+ "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={num_shard_bits=5;"
+ "high_pri_pool_ratio=0.0;}",
+ &new_opt));
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ 5);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ 5);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.0);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(new_opt.block_cache)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast<ShardedCacheBase>(
+ new_opt.block_cache_compressed)
+ ->GetNumShardBits(),
+ 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+}
+
+TEST_F(OptionsOldApiTest, GetPlainTableOptionsFromString) {
+ PlainTableOptions table_opt;
+ PlainTableOptions new_opt;
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
+ "full_scan_mode=true;store_index_in_file=true",
+ &new_opt));
+ ASSERT_EQ(new_opt.user_key_len, 66u);
+ ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
+ ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
+ ASSERT_EQ(new_opt.index_sparseness, 8);
+ ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
+ ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
+ ASSERT_TRUE(new_opt.full_scan_mode);
+ ASSERT_TRUE(new_opt.store_index_in_file);
+
+ std::unordered_map<std::string, std::string> opt_map;
+ ASSERT_OK(StringToMap(
+ "user_key_len=55;bloom_bits_per_key=10;huge_page_tlb_size=8;", &opt_map));
+ ASSERT_OK(GetPlainTableOptionsFromMap(table_opt, opt_map, &new_opt));
+ ASSERT_EQ(new_opt.user_key_len, 55u);
+ ASSERT_EQ(new_opt.bloom_bits_per_key, 10);
+ ASSERT_EQ(new_opt.huge_page_tlb_size, 8);
+
+ // unknown option
+ ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "bad_option=1",
+ &new_opt));
+
+ // unrecognized EncodingType
+ ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "encoding_type=kPrefixXX",
+ &new_opt));
+}
+
+TEST_F(OptionsOldApiTest, GetOptionsFromStringTest) {
+ Options base_options, new_options;
+ base_options.write_buffer_size = 20;
+ base_options.min_write_buffer_number_to_merge = 15;
+ BlockBasedTableOptions block_based_table_options;
+ block_based_table_options.cache_index_and_filter_blocks = true;
+ base_options.table_factory.reset(
+ NewBlockBasedTableFactory(block_based_table_options));
+
+ // Register an Env with object registry.
+ ObjectLibrary::Default()->AddFactory<Env>(
+ "CustomEnvDefault",
+ [](const std::string& /*name*/, std::unique_ptr<Env>* /*env_guard*/,
+ std::string* /* errmsg */) {
+ static CustomEnv env(Env::Default());
+ return &env;
+ });
+
+ ASSERT_OK(GetOptionsFromString(
+ base_options,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;"
+ "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files="
+ "1;"
+ "rate_limiter_bytes_per_sec=1024;env=CustomEnvDefault",
+ &new_options));
+
+ ASSERT_EQ(new_options.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_options.compression_opts.level, 5);
+ ASSERT_EQ(new_options.compression_opts.strategy, 6);
+ ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.parallel_threads, 1u);
+ ASSERT_EQ(new_options.compression_opts.enabled, false);
+ ASSERT_EQ(new_options.compression_opts.use_zstd_dict_trainer, true);
+ ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption);
+ ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_options.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.parallel_threads, 1u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
+ ASSERT_EQ(new_options.bottommost_compression_opts.use_zstd_dict_trainer,
+ true);
+ ASSERT_EQ(new_options.write_buffer_size, 10U);
+ ASSERT_EQ(new_options.max_write_buffer_number, 16);
+
+ auto new_block_based_table_options =
+ new_options.table_factory->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(new_block_based_table_options, nullptr);
+ ASSERT_EQ(new_block_based_table_options->block_cache->GetCapacity(),
+ 1U << 20);
+ ASSERT_EQ(new_block_based_table_options->block_size, 4U);
+ // don't overwrite block based table options
+ ASSERT_TRUE(new_block_based_table_options->cache_index_and_filter_blocks);
+
+ ASSERT_EQ(new_options.create_if_missing, true);
+ ASSERT_EQ(new_options.max_open_files, 1);
+ ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
+ Env* newEnv = new_options.env;
+ ASSERT_OK(Env::LoadEnv("CustomEnvDefault", &newEnv));
+ ASSERT_EQ(newEnv, new_options.env);
+}
+
+TEST_F(OptionsOldApiTest, DBOptionsSerialization) {
+ Options base_options, new_options;
+ Random rnd(301);
+
+ // Phase 1: Make big change in base_options
+ test::RandomInitDBOptions(&base_options, &rnd);
+
+ // Phase 2: obtain a string from base_option
+ std::string base_options_file_content;
+ ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options));
+
+ // Phase 3: Set new_options from the derived string and expect
+ // new_options == base_options
+ ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content,
+ &new_options));
+ ConfigOptions config_options;
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_options, new_options));
+}
+
+TEST_F(OptionsOldApiTest, ColumnFamilyOptionsSerialization) {
+ Options options;
+ ColumnFamilyOptions base_opt, new_opt;
+ Random rnd(302);
+ // Phase 1: randomly assign base_opt
+ // custom type options
+ test::RandomInitCFOptions(&base_opt, options, &rnd);
+
+ // Phase 2: obtain a string from base_opt
+ std::string base_options_file_content;
+ ASSERT_OK(
+ GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt));
+
+ // Phase 3: Set new_opt from the derived string and expect
+ // new_opt == base_opt
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ ColumnFamilyOptions(), base_options_file_content, &new_opt));
+ ConfigOptions config_options;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt));
+ if (base_opt.compaction_filter) {
+ delete base_opt.compaction_filter;
+ }
+}
+#endif // !ROCKSDB_LITE
+
+#ifndef ROCKSDB_LITE
+class OptionsParserTest : public testing::Test {
+ public:
+ OptionsParserTest() { fs_.reset(new test::StringFS(FileSystem::Default())); }
+
+ protected:
+ std::shared_ptr<test::StringFS> fs_;
+};
+
+TEST_F(OptionsParserTest, Comment) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[ DBOptions ]\n"
+ " # note that we don't support space around \"=\"\n"
+ " max_open_files=12345;\n"
+ " max_background_flushes=301 # comment after a statement is fine\n"
+ " # max_background_flushes=1000 # this line would be ignored\n"
+ " # max_background_compactions=2000 # so does this one\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"default\"] # column family must be specified\n"
+ " # in the correct order\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_OK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+
+ ConfigOptions exact;
+ exact.input_strings_escaped = false;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, *parser.db_opt(), db_opt));
+ ASSERT_EQ(parser.NumColumnFamilies(), 1U);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ exact, *parser.GetCFOptions("default"), cf_opt));
+}
+
+TEST_F(OptionsParserTest, ExtraSpace) {
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[ Version ]\n"
+ " rocksdb_version = 3.14.0 \n"
+ " options_file_version=1 # some comment\n"
+ "[DBOptions ] # some comment\n"
+ "max_open_files=12345 \n"
+ " max_background_flushes = 301 \n"
+ " max_total_wal_size = 1024 # keep_log_file_num=1000\n"
+ " [CFOptions \"default\" ]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_OK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, MissingDBOptions) {
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[CFOptions \"default\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+ ;
+}
+
+TEST_F(OptionsParserTest, DoubleDBOptions) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[DBOptions]\n"
+ "[CFOptions \"default\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, NoDefaultCFOptions) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"something_else\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"something_else\"]\n"
+ " # if a section is blank, we will use the default\n"
+ "[CFOptions \"default\"]\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, DuplicateCFOptions) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.14.0\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ "[CFOptions \"default\"]\n"
+ "[CFOptions \"something_else\"]\n"
+ "[CFOptions \"something_else\"]\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(
+ parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+}
+
+TEST_F(OptionsParserTest, IgnoreUnknownOptions) {
+ for (int case_id = 0; case_id < 5; case_id++) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string version_string;
+ bool should_ignore = true;
+ if (case_id == 0) {
+ // same version
+ should_ignore = false;
+ version_string = std::to_string(ROCKSDB_MAJOR) + "." +
+ std::to_string(ROCKSDB_MINOR) + ".0";
+ } else if (case_id == 1) {
+ // higher minor version
+ should_ignore = true;
+ version_string = std::to_string(ROCKSDB_MAJOR) + "." +
+ std::to_string(ROCKSDB_MINOR + 1) + ".0";
+ } else if (case_id == 2) {
+ // higher major version.
+ should_ignore = true;
+ version_string = std::to_string(ROCKSDB_MAJOR + 1) + ".0.0";
+ } else if (case_id == 3) {
+ // lower minor version
+#if ROCKSDB_MINOR == 0
+ continue;
+#else
+ version_string = std::to_string(ROCKSDB_MAJOR) + "." +
+ std::to_string(ROCKSDB_MINOR - 1) + ".0";
+ should_ignore = false;
+#endif
+ } else {
+ // lower major version
+ should_ignore = false;
+ version_string = std::to_string(ROCKSDB_MAJOR - 1) + "." +
+ std::to_string(ROCKSDB_MINOR) + ".0";
+ }
+
+ std::string options_file_content =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=" +
+ version_string +
+ "\n"
+ " options_file_version=1\n"
+ "[DBOptions]\n"
+ " max_open_files=12345\n"
+ " max_background_flushes=301\n"
+ " max_total_wal_size=1024 # keep_log_file_num=1000\n"
+ " unknown_db_option1=321\n"
+ " unknown_db_option2=false\n"
+ "[CFOptions \"default\"]\n"
+ " unknown_cf_option1=hello\n"
+ "[CFOptions \"something_else\"]\n"
+ " unknown_cf_option2=world\n"
+ " # if a section is blank, we will use the default\n";
+
+ const std::string kTestFileName = "test-rocksdb-options.ini";
+ auto s = fs_->FileExists(kTestFileName, IOOptions(), nullptr);
+ ASSERT_TRUE(s.ok() || s.IsNotFound());
+ if (s.ok()) {
+ ASSERT_OK(fs_->DeleteFile(kTestFileName, IOOptions(), nullptr));
+ }
+ ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
+ RocksDBOptionsParser parser;
+ ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), false,
+ 4096 /* readahead_size */));
+ if (should_ignore) {
+ ASSERT_OK(parser.Parse(kTestFileName, fs_.get(),
+ true /* ignore_unknown_options */,
+ 4096 /* readahead_size */));
+ } else {
+ ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(),
+ true /* ignore_unknown_options */,
+ 4096 /* readahead_size */));
+ }
+ }
+}
+
+TEST_F(OptionsParserTest, ParseVersion) {
+ DBOptions db_opt;
+ db_opt.max_open_files = 12345;
+ db_opt.max_background_flushes = 301;
+ db_opt.max_total_wal_size = 1024;
+ ColumnFamilyOptions cf_opt;
+
+ std::string file_template =
+ "# This is a testing option string.\n"
+ "# Currently we only support \"#\" styled comment.\n"
+ "\n"
+ "[Version]\n"
+ " rocksdb_version=3.13.1\n"
+ " options_file_version=%s\n"
+ "[DBOptions]\n"
+ "[CFOptions \"default\"]\n";
+ const int kLength = 1000;
+ char buffer[kLength];
+ RocksDBOptionsParser parser;
+
+ const std::vector<std::string> invalid_versions = {
+ "a.b.c", "3.2.2b", "3.-12", "3. 1", // only digits and dots are allowed
+ "1.2.3.4",
+ "1.2.3" // can only contains at most one dot.
+ "0", // options_file_version must be at least one
+ "3..2",
+ ".", ".1.2", // must have at least one digit before each dot
+ "1.2.", "1.", "2.34."}; // must have at least one digit after each dot
+ for (auto iv : invalid_versions) {
+ snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str());
+
+ parser.Reset();
+ ASSERT_OK(fs_->WriteToNewFile(iv, buffer));
+ ASSERT_NOK(parser.Parse(iv, fs_.get(), false, 0 /* readahead_size */));
+ }
+
+ const std::vector<std::string> valid_versions = {
+ "1.232", "100", "3.12", "1", "12.3 ", " 1.25 "};
+ for (auto vv : valid_versions) {
+ snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str());
+ parser.Reset();
+ ASSERT_OK(fs_->WriteToNewFile(vv, buffer));
+ ASSERT_OK(parser.Parse(vv, fs_.get(), false, 0 /* readahead_size */));
+ }
+}
+
+void VerifyCFPointerTypedOptions(
+ ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
+ const std::unordered_map<std::string, std::string>* new_cf_opt_map) {
+ std::string name_buffer;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, *base_cf_opt,
+ *new_cf_opt, new_cf_opt_map));
+
+ // change the name of merge operator back-and-forth
+ {
+ auto* merge_operator = base_cf_opt->merge_operator
+ ->CheckedCast<test::ChanglingMergeOperator>();
+ if (merge_operator != nullptr) {
+ name_buffer = merge_operator->Name();
+ // change the name and expect non-ok status
+ merge_operator->SetName("some-other-name");
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // change the name back and expect ok status
+ merge_operator->SetName(name_buffer);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ }
+ }
+
+ // change the name of the compaction filter factory back-and-forth
+ {
+ auto* compaction_filter_factory =
+ base_cf_opt->compaction_filter_factory
+ ->CheckedCast<test::ChanglingCompactionFilterFactory>();
+ if (compaction_filter_factory != nullptr) {
+ name_buffer = compaction_filter_factory->Name();
+ // change the name and expect non-ok status
+ compaction_filter_factory->SetName("some-other-name");
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // change the name back and expect ok status
+ compaction_filter_factory->SetName(name_buffer);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ }
+ }
+
+ // test by setting compaction_filter to nullptr
+ {
+ auto* tmp_compaction_filter = base_cf_opt->compaction_filter;
+ if (tmp_compaction_filter != nullptr) {
+ base_cf_opt->compaction_filter = nullptr;
+ // set compaction_filter to nullptr and expect non-ok status
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // set the value back and expect ok status
+ base_cf_opt->compaction_filter = tmp_compaction_filter;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ }
+ }
+
+ // test by setting table_factory to nullptr
+ {
+ auto tmp_table_factory = base_cf_opt->table_factory;
+ if (tmp_table_factory != nullptr) {
+ base_cf_opt->table_factory.reset();
+ // set table_factory to nullptr and expect non-ok status
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // set the value back and expect ok status
+ base_cf_opt->table_factory = tmp_table_factory;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ }
+ }
+
+ // test by setting memtable_factory to nullptr
+ {
+ auto tmp_memtable_factory = base_cf_opt->memtable_factory;
+ if (tmp_memtable_factory != nullptr) {
+ base_cf_opt->memtable_factory.reset();
+ // set memtable_factory to nullptr and expect non-ok status
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ // set the value back and expect ok status
+ base_cf_opt->memtable_factory = tmp_memtable_factory;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ }
+ }
+}
+
+TEST_F(OptionsParserTest, Readahead) {
+ DBOptions base_db_opt;
+ std::vector<ColumnFamilyOptions> base_cf_opts;
+ base_cf_opts.emplace_back();
+ base_cf_opts.emplace_back();
+
+ std::string one_mb_string = std::string(1024 * 1024, 'x');
+ std::vector<std::string> cf_names = {"default", one_mb_string};
+ const std::string kOptionsFileName = "test-persisted-options.ini";
+
+ ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
+ kOptionsFileName, fs_.get()));
+
+ uint64_t file_size = 0;
+ ASSERT_OK(
+ fs_->GetFileSize(kOptionsFileName, IOOptions(), &file_size, nullptr));
+ assert(file_size > 0);
+
+ RocksDBOptionsParser parser;
+
+ fs_->num_seq_file_read_ = 0;
+ size_t readahead_size = 128 * 1024;
+
+ ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
+ ASSERT_EQ(fs_->num_seq_file_read_.load(),
+ (file_size - 1) / readahead_size + 1);
+
+ fs_->num_seq_file_read_.store(0);
+ readahead_size = 1024 * 1024;
+ ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
+ ASSERT_EQ(fs_->num_seq_file_read_.load(),
+ (file_size - 1) / readahead_size + 1);
+
+ // Tiny readahead. 8 KB is read each time.
+ fs_->num_seq_file_read_.store(0);
+ ASSERT_OK(
+ parser.Parse(kOptionsFileName, fs_.get(), false, 1 /* readahead_size */));
+ ASSERT_GE(fs_->num_seq_file_read_.load(), file_size / (8 * 1024));
+ ASSERT_LT(fs_->num_seq_file_read_.load(), file_size / (8 * 1024) * 2);
+
+ // Disable readahead means 512KB readahead.
+ fs_->num_seq_file_read_.store(0);
+ ASSERT_OK(
+ parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
+ ASSERT_GE(fs_->num_seq_file_read_.load(), (file_size - 1) / (512 * 1024) + 1);
+}
+
+TEST_F(OptionsParserTest, DumpAndParse) {
+ DBOptions base_db_opt;
+ std::vector<ColumnFamilyOptions> base_cf_opts;
+ std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3",
+ "c:f:4:4:4"
+ "p\\i\\k\\a\\chu\\\\\\",
+ "###rocksdb#1-testcf#2###"};
+ const int num_cf = static_cast<int>(cf_names.size());
+ Random rnd(302);
+ test::RandomInitDBOptions(&base_db_opt, &rnd);
+ base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG";
+
+ BlockBasedTableOptions special_bbto;
+ special_bbto.cache_index_and_filter_blocks = true;
+ special_bbto.block_size = 999999;
+
+ for (int c = 0; c < num_cf; ++c) {
+ ColumnFamilyOptions cf_opt;
+ Random cf_rnd(0xFB + c);
+ test::RandomInitCFOptions(&cf_opt, base_db_opt, &cf_rnd);
+ if (c < 4) {
+ cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c));
+ }
+ if (c < 3) {
+ cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c));
+ } else if (c == 4) {
+ cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto));
+ } else if (c == 5) {
+ // A table factory that doesn't support deserialization should be
+ // supported.
+ cf_opt.table_factory.reset(new UnregisteredTableFactory());
+ }
+ base_cf_opts.emplace_back(cf_opt);
+ }
+
+ const std::string kOptionsFileName = "test-persisted-options.ini";
+ // Use default for escaped(true), unknown(false) and check (exact)
+ ConfigOptions config_options;
+ ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
+ kOptionsFileName, fs_.get()));
+
+ RocksDBOptionsParser parser;
+ ASSERT_OK(parser.Parse(config_options, kOptionsFileName, fs_.get()));
+
+ // Make sure block-based table factory options was deserialized correctly
+ std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory;
+ ASSERT_EQ(TableFactory::kBlockBasedTableName(), std::string(ttf->Name()));
+ const auto parsed_bbto = ttf->GetOptions<BlockBasedTableOptions>();
+ ASSERT_NE(parsed_bbto, nullptr);
+ ASSERT_EQ(special_bbto.block_size, parsed_bbto->block_size);
+ ASSERT_EQ(special_bbto.cache_index_and_filter_blocks,
+ parsed_bbto->cache_index_and_filter_blocks);
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName,
+ fs_.get()));
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
+ config_options, *parser.db_opt(), base_db_opt));
+ for (int c = 0; c < num_cf; ++c) {
+ const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
+ ASSERT_NE(cf_opt, nullptr);
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, base_cf_opts[c], *cf_opt,
+ &(parser.cf_opt_maps()->at(c))));
+ }
+
+ // Further verify pointer-typed options
+ for (int c = 0; c < num_cf; ++c) {
+ const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
+ ASSERT_NE(cf_opt, nullptr);
+ VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt,
+ &(parser.cf_opt_maps()->at(c)));
+ }
+
+ ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr);
+
+ base_db_opt.max_open_files++;
+ ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName,
+ fs_.get()));
+
+ for (int c = 0; c < num_cf; ++c) {
+ if (base_cf_opts[c].compaction_filter) {
+ delete base_cf_opts[c].compaction_filter;
+ }
+ }
+}
+
+TEST_F(OptionsParserTest, DifferentDefault) {
+ const std::string kOptionsFileName = "test-persisted-options.ini";
+
+ ColumnFamilyOptions cf_level_opts;
+ ASSERT_EQ(CompactionPri::kMinOverlappingRatio, cf_level_opts.compaction_pri);
+ cf_level_opts.OptimizeLevelStyleCompaction();
+
+ ColumnFamilyOptions cf_univ_opts;
+ cf_univ_opts.OptimizeUniversalStyleCompaction();
+
+ ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"},
+ {cf_level_opts, cf_univ_opts},
+ kOptionsFileName, fs_.get()));
+
+ RocksDBOptionsParser parser;
+ ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false,
+ 4096 /* readahead_size */));
+
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults();
+ ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
+ ASSERT_EQ(5000, old_default_opts.max_open_files);
+ ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ ASSERT_EQ(WALRecoveryMode::kTolerateCorruptedTailRecords,
+ old_default_opts.wal_recovery_mode);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(4, 6);
+ ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
+ ASSERT_EQ(5000, old_default_opts.max_open_files);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(4, 7);
+ ASSERT_NE(10 * 1048576, old_default_opts.max_bytes_for_level_base);
+ ASSERT_NE(4, old_default_opts.table_cache_numshardbits);
+ ASSERT_EQ(5000, old_default_opts.max_open_files);
+ ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ }
+ {
+ ColumnFamilyOptions old_default_cf_opts;
+ old_default_cf_opts.OldDefaults();
+ ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(4 << 20, old_default_cf_opts.write_buffer_size);
+ ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(0, old_default_cf_opts.soft_pending_compaction_bytes_limit);
+ ASSERT_EQ(0, old_default_cf_opts.hard_pending_compaction_bytes_limit);
+ ASSERT_EQ(CompactionPri::kByCompensatedSize,
+ old_default_cf_opts.compaction_pri);
+ }
+ {
+ ColumnFamilyOptions old_default_cf_opts;
+ old_default_cf_opts.OldDefaults(4, 6);
+ ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(CompactionPri::kByCompensatedSize,
+ old_default_cf_opts.compaction_pri);
+ }
+ {
+ ColumnFamilyOptions old_default_cf_opts;
+ old_default_cf_opts.OldDefaults(4, 7);
+ ASSERT_NE(2 * 1048576, old_default_cf_opts.target_file_size_base);
+ ASSERT_EQ(CompactionPri::kByCompensatedSize,
+ old_default_cf_opts.compaction_pri);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(5, 1);
+ ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(5, 2);
+ ASSERT_EQ(16 * 1024U * 1024U, old_default_opts.delayed_write_rate);
+ ASSERT_TRUE(old_default_opts.compaction_pri ==
+ CompactionPri::kByCompensatedSize);
+ }
+ {
+ Options old_default_opts;
+ old_default_opts.OldDefaults(5, 18);
+ ASSERT_TRUE(old_default_opts.compaction_pri ==
+ CompactionPri::kByCompensatedSize);
+ }
+
+ Options small_opts;
+ small_opts.OptimizeForSmallDb();
+ ASSERT_EQ(2 << 20, small_opts.write_buffer_size);
+ ASSERT_EQ(5000, small_opts.max_open_files);
+}
+
+class OptionsSanityCheckTest : public OptionsParserTest,
+ public ::testing::WithParamInterface<bool> {
+ protected:
+ ConfigOptions config_options_;
+
+ public:
+ OptionsSanityCheckTest() {
+ config_options_.ignore_unknown_options = false;
+ config_options_.ignore_unsupported_options = GetParam();
+ config_options_.input_strings_escaped = true;
+ }
+
+ protected:
+ Status SanityCheckOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts,
+ ConfigOptions::SanityLevel level) {
+ config_options_.sanity_level = level;
+ return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+ config_options_, db_opts, {"default"}, {cf_opts}, kOptionsFileName,
+ fs_.get());
+ }
+
+ Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts,
+ ConfigOptions::SanityLevel level) {
+ return SanityCheckOptions(DBOptions(), cf_opts, level);
+ }
+
+ void SanityCheckCFOptions(const ColumnFamilyOptions& opts, bool exact) {
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+ if (exact) {
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ } else {
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ }
+ }
+
+ Status SanityCheckDBOptions(const DBOptions& db_opts,
+ ConfigOptions::SanityLevel level) {
+ return SanityCheckOptions(db_opts, ColumnFamilyOptions(), level);
+ }
+
+ void SanityCheckDBOptions(const DBOptions& opts, bool exact) {
+ ASSERT_OK(SanityCheckDBOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelNone));
+ if (exact) {
+ ASSERT_OK(
+ SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ } else {
+ ASSERT_NOK(
+ SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ }
+ }
+
+ Status PersistOptions(const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts) {
+ Status s = fs_->DeleteFile(kOptionsFileName, IOOptions(), nullptr);
+ if (!s.ok()) {
+ return s;
+ }
+ return PersistRocksDBOptions(db_opts, {"default"}, {cf_opts},
+ kOptionsFileName, fs_.get());
+ }
+
+ Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) {
+ return PersistOptions(DBOptions(), cf_opts);
+ }
+
+ Status PersistDBOptions(const DBOptions& db_opts) {
+ return PersistOptions(db_opts, ColumnFamilyOptions());
+ }
+
+ const std::string kOptionsFileName = "OPTIONS";
+};
+
+TEST_P(OptionsSanityCheckTest, CFOptionsSanityCheck) {
+ ColumnFamilyOptions opts;
+ Random rnd(301);
+
+ // default ColumnFamilyOptions
+ {
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ }
+
+ // prefix_extractor
+ {
+ // Okay to change prefix_extractor form nullptr to non-nullptr
+ ASSERT_EQ(opts.prefix_extractor.get(), nullptr);
+ opts.prefix_extractor.reset(NewCappedPrefixTransform(10));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+
+ // use same prefix extractor but with different parameter
+ opts.prefix_extractor.reset(NewCappedPrefixTransform(15));
+ // expect pass only in
+ // ConfigOptions::kSanityLevelLooselyCompatible
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+
+ // repeat the test with FixedPrefixTransform
+ opts.prefix_extractor.reset(NewFixedPrefixTransform(10));
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+
+ // persist the change of prefix_extractor
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+
+ // use same prefix extractor but with different parameter
+ opts.prefix_extractor.reset(NewFixedPrefixTransform(15));
+ // expect pass only in
+ // ConfigOptions::kSanityLevelLooselyCompatible
+ SanityCheckCFOptions(opts, false);
+
+ // Change prefix extractor from non-nullptr to nullptr
+ opts.prefix_extractor.reset();
+ // expect pass as it's safe to change prefix_extractor
+ // from non-null to null
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+ }
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+
+ // table_factory
+ {
+ for (int tb = 0; tb <= 2; ++tb) {
+ // change the table factory
+ opts.table_factory.reset(test::RandomTableFactory(&rnd, tb));
+ ASSERT_NOK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ }
+ }
+
+ // merge_operator
+ {
+ // Test when going from nullptr -> merge operator
+ opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options);
+
+ for (int test = 0; test < 5; ++test) {
+ // change the merge operator
+ opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
+ ASSERT_NOK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options);
+ }
+
+ // Test when going from merge operator -> nullptr
+ opts.merge_operator = nullptr;
+ ASSERT_NOK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ SanityCheckCFOptions(opts, true);
+ }
+
+ // compaction_filter
+ {
+ for (int test = 0; test < 5; ++test) {
+ // change the compaction filter
+ opts.compaction_filter = test::RandomCompactionFilter(&rnd);
+ SanityCheckCFOptions(opts, false);
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options);
+ delete opts.compaction_filter;
+ opts.compaction_filter = nullptr;
+ }
+ }
+
+ // compaction_filter_factory
+ {
+ for (int test = 0; test < 5; ++test) {
+ // change the compaction filter factory
+ opts.compaction_filter_factory.reset(
+ test::RandomCompactionFilterFactory(&rnd));
+ SanityCheckCFOptions(opts, false);
+
+ // persist the change
+ ASSERT_OK(PersistCFOptions(opts));
+ SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options);
+ }
+ }
+}
+
+TEST_P(OptionsSanityCheckTest, DBOptionsSanityCheck) {
+ DBOptions opts;
+ Random rnd(301);
+
+ // default DBOptions
+ {
+ ASSERT_OK(PersistDBOptions(opts));
+ ASSERT_OK(
+ SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ }
+
+ // File checksum generator
+ {
+ class MockFileChecksumGenFactory : public FileChecksumGenFactory {
+ public:
+ static const char* kClassName() { return "Mock"; }
+ const char* Name() const override { return kClassName(); }
+ std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
+ const FileChecksumGenContext& /*context*/) override {
+ return nullptr;
+ }
+ };
+
+ // Okay to change file_checksum_gen_factory form nullptr to non-nullptr
+ ASSERT_EQ(opts.file_checksum_gen_factory.get(), nullptr);
+ opts.file_checksum_gen_factory.reset(new MockFileChecksumGenFactory());
+
+ // persist the change
+ ASSERT_OK(PersistDBOptions(opts));
+ SanityCheckDBOptions(opts, config_options_.ignore_unsupported_options);
+
+ // Change file_checksum_gen_factory from non-nullptr to nullptr
+ opts.file_checksum_gen_factory.reset();
+ // expect pass as it's safe to change file_checksum_gen_factory
+ // from non-null to null
+ SanityCheckDBOptions(opts, false);
+ }
+ // persist the change
+ ASSERT_OK(PersistDBOptions(opts));
+ ASSERT_OK(SanityCheckDBOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+}
+
+namespace {
+bool IsEscapedString(const std::string& str) {
+ for (size_t i = 0; i < str.size(); ++i) {
+ if (str[i] == '\\') {
+ // since we already handle those two consecutive '\'s in
+ // the next if-then branch, any '\' appear at the end
+ // of an escaped string in such case is not valid.
+ if (i == str.size() - 1) {
+ return false;
+ }
+ if (str[i + 1] == '\\') {
+ // if there're two consecutive '\'s, skip the second one.
+ i++;
+ continue;
+ }
+ switch (str[i + 1]) {
+ case ':':
+ case '\\':
+ case '#':
+ continue;
+ default:
+ // if true, '\' together with str[i + 1] is not a valid escape.
+ if (UnescapeChar(str[i + 1]) == str[i + 1]) {
+ return false;
+ }
+ }
+ } else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) {
+ return false;
+ }
+ }
+ return true;
+}
+} // namespace
+
+TEST_F(OptionsParserTest, IntegerParsing) {
+ ASSERT_EQ(ParseUint64("18446744073709551615"), 18446744073709551615U);
+ ASSERT_EQ(ParseUint32("4294967295"), 4294967295U);
+ ASSERT_EQ(ParseSizeT("18446744073709551615"), 18446744073709551615U);
+ ASSERT_EQ(ParseInt64("9223372036854775807"), 9223372036854775807);
+ ASSERT_EQ(ParseInt64("-9223372036854775808"),
+ std::numeric_limits<int64_t>::min());
+ ASSERT_EQ(ParseInt32("2147483647"), 2147483647);
+ ASSERT_EQ(ParseInt32("-2147483648"), std::numeric_limits<int32_t>::min());
+ ASSERT_EQ(ParseInt("-32767"), -32767);
+ ASSERT_EQ(ParseDouble("-1.234567"), -1.234567);
+}
+
+TEST_F(OptionsParserTest, EscapeOptionString) {
+ ASSERT_EQ(UnescapeOptionString(
+ "This is a test string with \\# \\: and \\\\ escape chars."),
+ "This is a test string with # : and \\ escape chars.");
+
+ ASSERT_EQ(
+ EscapeOptionString("This is a test string with # : and \\ escape chars."),
+ "This is a test string with \\# \\: and \\\\ escape chars.");
+
+ std::string readible_chars =
+ "A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU"
+ "YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>"
+ "<MNBVCXZ\\\" should be okay to \\#\\\\\\:\\#\\#\\#\\ "
+ "be serialized and deserialized";
+
+ std::string escaped_string = EscapeOptionString(readible_chars);
+ ASSERT_TRUE(IsEscapedString(escaped_string));
+ // This two transformations should be canceled and should output
+ // the original input.
+ ASSERT_EQ(UnescapeOptionString(escaped_string), readible_chars);
+
+ std::string all_chars;
+ for (unsigned char c = 0;; ++c) {
+ all_chars += c;
+ if (c == 255) {
+ break;
+ }
+ }
+ escaped_string = EscapeOptionString(all_chars);
+ ASSERT_TRUE(IsEscapedString(escaped_string));
+ ASSERT_EQ(UnescapeOptionString(escaped_string), all_chars);
+
+ ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
+ " A simple statement with a comment. # like this :)"),
+ "A simple statement with a comment.");
+
+ ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
+ "Escape \\# and # comment together ."),
+ "Escape \\# and");
+}
+
+static void TestAndCompareOption(const ConfigOptions& config_options,
+ const OptionTypeInfo& opt_info,
+ const std::string& opt_name, void* base_ptr,
+ void* comp_ptr, bool strip = false) {
+ std::string result, mismatch;
+ ASSERT_OK(opt_info.Serialize(config_options, opt_name, base_ptr, &result));
+ if (strip) {
+ ASSERT_EQ(result.at(0), '{');
+ ASSERT_EQ(result.at(result.size() - 1), '}');
+ result = result.substr(1, result.size() - 2);
+ }
+ ASSERT_OK(opt_info.Parse(config_options, opt_name, result, comp_ptr));
+ ASSERT_TRUE(opt_info.AreEqual(config_options, opt_name, base_ptr, comp_ptr,
+ &mismatch));
+}
+
+static void TestParseAndCompareOption(const ConfigOptions& config_options,
+ const OptionTypeInfo& opt_info,
+ const std::string& opt_name,
+ const std::string& opt_value,
+ void* base_ptr, void* comp_ptr,
+ bool strip = false) {
+ ASSERT_OK(opt_info.Parse(config_options, opt_name, opt_value, base_ptr));
+ TestAndCompareOption(config_options, opt_info, opt_name, base_ptr, comp_ptr,
+ strip);
+}
+
+template <typename T>
+void TestOptInfo(const ConfigOptions& config_options, OptionType opt_type,
+ T* base, T* comp) {
+ std::string result;
+ OptionTypeInfo opt_info(0, opt_type);
+ ASSERT_FALSE(opt_info.AreEqual(config_options, "base", base, comp, &result));
+ ASSERT_EQ(result, "base");
+ ASSERT_NE(*base, *comp);
+ TestAndCompareOption(config_options, opt_info, "base", base, comp);
+ ASSERT_EQ(*base, *comp);
+}
+
+class OptionTypeInfoTest : public testing::Test {};
+
+TEST_F(OptionTypeInfoTest, BasicTypes) {
+ ConfigOptions config_options;
+ {
+ bool a = true, b = false;
+ TestOptInfo(config_options, OptionType::kBoolean, &a, &b);
+ }
+ {
+ int a = 100, b = 200;
+ TestOptInfo(config_options, OptionType::kInt, &a, &b);
+ }
+ {
+ int32_t a = 100, b = 200;
+ TestOptInfo(config_options, OptionType::kInt32T, &a, &b);
+ }
+ {
+ int64_t a = 100, b = 200;
+ TestOptInfo(config_options, OptionType::kInt64T, &a, &b);
+ }
+ {
+ unsigned int a = 100, b = 200;
+ TestOptInfo(config_options, OptionType::kUInt, &a, &b);
+ }
+ {
+ uint32_t a = 100, b = 200;
+ TestOptInfo(config_options, OptionType::kUInt32T, &a, &b);
+ }
+ {
+ uint64_t a = 100, b = 200;
+ TestOptInfo(config_options, OptionType::kUInt64T, &a, &b);
+ }
+ {
+ size_t a = 100, b = 200;
+ TestOptInfo(config_options, OptionType::kSizeT, &a, &b);
+ }
+ {
+ std::string a = "100", b = "200";
+ TestOptInfo(config_options, OptionType::kString, &a, &b);
+ }
+ {
+ double a = 1.0, b = 2.0;
+ TestOptInfo(config_options, OptionType::kDouble, &a, &b);
+ }
+}
+
+TEST_F(OptionTypeInfoTest, TestInvalidArgs) {
+ ConfigOptions config_options;
+ bool b;
+ int i;
+ int32_t i32;
+ int64_t i64;
+ unsigned int u;
+ int32_t u32;
+ int64_t u64;
+ size_t sz;
+ double d;
+
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kBoolean)
+ .Parse(config_options, "b", "x", &b));
+ ASSERT_NOK(
+ OptionTypeInfo(0, OptionType::kInt).Parse(config_options, "b", "x", &i));
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt32T)
+ .Parse(config_options, "b", "x", &i32));
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt64T)
+ .Parse(config_options, "b", "x", &i64));
+ ASSERT_NOK(
+ OptionTypeInfo(0, OptionType::kUInt).Parse(config_options, "b", "x", &u));
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt32T)
+ .Parse(config_options, "b", "x", &u32));
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt64T)
+ .Parse(config_options, "b", "x", &u64));
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kSizeT)
+ .Parse(config_options, "b", "x", &sz));
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kDouble)
+ .Parse(config_options, "b", "x", &d));
+
+ // Don't know how to convert Unknowns to anything else
+ ASSERT_NOK(OptionTypeInfo(0, OptionType::kUnknown)
+ .Parse(config_options, "b", "x", &d));
+
+ // Verify that if the parse function throws an exception, it is also trapped
+ OptionTypeInfo func_info(0, OptionType::kUnknown,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone,
+ [](const ConfigOptions&, const std::string&,
+ const std::string& value, void* addr) {
+ auto ptr = static_cast<int*>(addr);
+ *ptr = ParseInt(value);
+ return Status::OK();
+ });
+ ASSERT_OK(func_info.Parse(config_options, "b", "1", &i));
+ ASSERT_NOK(func_info.Parse(config_options, "b", "x", &i));
+}
+
+TEST_F(OptionTypeInfoTest, TestParseFunc) {
+ OptionTypeInfo opt_info(0, OptionType::kUnknown,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone);
+ opt_info.SetParseFunc([](const ConfigOptions& /*opts*/,
+ const std::string& name, const std::string& value,
+ void* addr) {
+ auto ptr = static_cast<std::string*>(addr);
+ if (name == "Oops") {
+ return Status::InvalidArgument(value);
+ } else {
+ *ptr = value + " " + name;
+ return Status::OK();
+ }
+ });
+ ConfigOptions config_options;
+ std::string base;
+ ASSERT_OK(opt_info.Parse(config_options, "World", "Hello", &base));
+ ASSERT_EQ(base, "Hello World");
+ ASSERT_NOK(opt_info.Parse(config_options, "Oops", "Hello", &base));
+}
+
+TEST_F(OptionTypeInfoTest, TestSerializeFunc) {
+ OptionTypeInfo opt_info(0, OptionType::kString,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone);
+ opt_info.SetSerializeFunc([](const ConfigOptions& /*opts*/,
+ const std::string& name, const void* /*addr*/,
+ std::string* value) {
+ if (name == "Oops") {
+ return Status::InvalidArgument(name);
+ } else {
+ *value = name;
+ return Status::OK();
+ }
+ });
+ ConfigOptions config_options;
+ std::string base;
+ std::string value;
+ ASSERT_OK(opt_info.Serialize(config_options, "Hello", &base, &value));
+ ASSERT_EQ(value, "Hello");
+ ASSERT_NOK(opt_info.Serialize(config_options, "Oops", &base, &value));
+}
+
+TEST_F(OptionTypeInfoTest, TestEqualsFunc) {
+ OptionTypeInfo opt_info(0, OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone);
+ opt_info.SetEqualsFunc([](const ConfigOptions& /*opts*/,
+ const std::string& name, const void* addr1,
+ const void* addr2, std::string* mismatch) {
+ auto i1 = *(static_cast<const int*>(addr1));
+ auto i2 = *(static_cast<const int*>(addr2));
+ if (name == "LT") {
+ return i1 < i2;
+ } else if (name == "GT") {
+ return i1 > i2;
+ } else if (name == "EQ") {
+ return i1 == i2;
+ } else {
+ *mismatch = name + "???";
+ return false;
+ }
+ });
+
+ ConfigOptions config_options;
+ int int1 = 100;
+ int int2 = 200;
+ std::string mismatch;
+ ASSERT_TRUE(opt_info.AreEqual(config_options, "LT", &int1, &int2, &mismatch));
+ ASSERT_EQ(mismatch, "");
+ ASSERT_FALSE(
+ opt_info.AreEqual(config_options, "GT", &int1, &int2, &mismatch));
+ ASSERT_EQ(mismatch, "GT");
+ ASSERT_FALSE(
+ opt_info.AreEqual(config_options, "NO", &int1, &int2, &mismatch));
+ ASSERT_EQ(mismatch, "NO???");
+}
+
+TEST_F(OptionTypeInfoTest, TestPrepareFunc) {
+ OptionTypeInfo opt_info(0, OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone);
+ opt_info.SetPrepareFunc(
+ [](const ConfigOptions& /*opts*/, const std::string& name, void* addr) {
+ auto i1 = static_cast<int*>(addr);
+ if (name == "x2") {
+ *i1 *= 2;
+ } else if (name == "/2") {
+ *i1 /= 2;
+ } else {
+ return Status::InvalidArgument("Bad Argument", name);
+ }
+ return Status::OK();
+ });
+ ConfigOptions config_options;
+ int int1 = 100;
+ ASSERT_OK(opt_info.Prepare(config_options, "x2", &int1));
+ ASSERT_EQ(int1, 200);
+ ASSERT_OK(opt_info.Prepare(config_options, "/2", &int1));
+ ASSERT_EQ(int1, 100);
+ ASSERT_NOK(opt_info.Prepare(config_options, "??", &int1));
+ ASSERT_EQ(int1, 100);
+}
+TEST_F(OptionTypeInfoTest, TestValidateFunc) {
+ OptionTypeInfo opt_info(0, OptionType::kSizeT,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone);
+ opt_info.SetValidateFunc([](const DBOptions& db_opts,
+ const ColumnFamilyOptions& cf_opts,
+ const std::string& name, const void* addr) {
+ const auto sz = static_cast<const size_t*>(addr);
+ bool is_valid = false;
+ if (name == "keep_log_file_num") {
+ is_valid = (*sz == db_opts.keep_log_file_num);
+ } else if (name == "write_buffer_size") {
+ is_valid = (*sz == cf_opts.write_buffer_size);
+ }
+ if (is_valid) {
+ return Status::OK();
+ } else {
+ return Status::InvalidArgument("Mismatched value", name);
+ }
+ });
+ ConfigOptions config_options;
+ DBOptions db_options;
+ ColumnFamilyOptions cf_options;
+
+ ASSERT_OK(opt_info.Validate(db_options, cf_options, "keep_log_file_num",
+ &db_options.keep_log_file_num));
+ ASSERT_OK(opt_info.Validate(db_options, cf_options, "write_buffer_size",
+ &cf_options.write_buffer_size));
+ ASSERT_NOK(opt_info.Validate(db_options, cf_options, "keep_log_file_num",
+ &cf_options.write_buffer_size));
+ ASSERT_NOK(opt_info.Validate(db_options, cf_options, "write_buffer_size",
+ &db_options.keep_log_file_num));
+}
+
+TEST_F(OptionTypeInfoTest, TestOptionFlags) {
+ OptionTypeInfo opt_none(0, OptionType::kString,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kDontSerialize);
+ OptionTypeInfo opt_never(0, OptionType::kString,
+ OptionVerificationType::kNormal,
+ OptionTypeFlags::kCompareNever);
+ OptionTypeInfo opt_alias(0, OptionType::kString,
+ OptionVerificationType::kAlias,
+ OptionTypeFlags::kNone);
+ OptionTypeInfo opt_deprecated(0, OptionType::kString,
+ OptionVerificationType::kDeprecated,
+ OptionTypeFlags::kNone);
+ ConfigOptions config_options;
+ std::string opts_str;
+ std::string base = "base";
+ std::string comp = "comp";
+
+ // If marked string none, the serialization returns not supported
+ ASSERT_NOK(opt_none.Serialize(config_options, "None", &base, &opts_str));
+ // If marked never compare, they match even when they do not
+ ASSERT_TRUE(opt_never.AreEqual(config_options, "Never", &base, &comp, &base));
+ ASSERT_FALSE(opt_none.AreEqual(config_options, "Never", &base, &comp, &base));
+
+ // An alias can change the value via parse, but does nothing on serialize on
+ // match
+ std::string result;
+ ASSERT_OK(opt_alias.Parse(config_options, "Alias", "Alias", &base));
+ ASSERT_OK(opt_alias.Serialize(config_options, "Alias", &base, &result));
+ ASSERT_TRUE(
+ opt_alias.AreEqual(config_options, "Alias", &base, &comp, &result));
+ ASSERT_EQ(base, "Alias");
+ ASSERT_NE(base, comp);
+
+ // Deprecated options do nothing on any of the commands
+ ASSERT_OK(opt_deprecated.Parse(config_options, "Alias", "Deprecated", &base));
+ ASSERT_OK(opt_deprecated.Serialize(config_options, "Alias", &base, &result));
+ ASSERT_TRUE(
+ opt_deprecated.AreEqual(config_options, "Alias", &base, &comp, &result));
+ ASSERT_EQ(base, "Alias");
+ ASSERT_NE(base, comp);
+}
+
+TEST_F(OptionTypeInfoTest, TestCustomEnum) {
+ enum TestEnum { kA, kB, kC };
+ std::unordered_map<std::string, TestEnum> enum_map = {
+ {"A", TestEnum::kA},
+ {"B", TestEnum::kB},
+ {"C", TestEnum::kC},
+ };
+ OptionTypeInfo opt_info = OptionTypeInfo::Enum<TestEnum>(0, &enum_map);
+ TestEnum e1, e2;
+ ConfigOptions config_options;
+ std::string result, mismatch;
+
+ e2 = TestEnum::kA;
+
+ ASSERT_OK(opt_info.Parse(config_options, "", "B", &e1));
+ ASSERT_OK(opt_info.Serialize(config_options, "", &e1, &result));
+ ASSERT_EQ(e1, TestEnum::kB);
+ ASSERT_EQ(result, "B");
+
+ ASSERT_FALSE(opt_info.AreEqual(config_options, "Enum", &e1, &e2, &mismatch));
+ ASSERT_EQ(mismatch, "Enum");
+
+ TestParseAndCompareOption(config_options, opt_info, "", "C", &e1, &e2);
+ ASSERT_EQ(e2, TestEnum::kC);
+
+ ASSERT_NOK(opt_info.Parse(config_options, "", "D", &e1));
+ ASSERT_EQ(e1, TestEnum::kC);
+}
+
+TEST_F(OptionTypeInfoTest, TestBuiltinEnum) {
+ ConfigOptions config_options;
+ for (auto iter : OptionsHelper::compaction_style_string_map) {
+ CompactionStyle e1, e2;
+ TestParseAndCompareOption(config_options,
+ OptionTypeInfo(0, OptionType::kCompactionStyle),
+ "CompactionStyle", iter.first, &e1, &e2);
+ ASSERT_EQ(e1, iter.second);
+ }
+ for (auto iter : OptionsHelper::compaction_pri_string_map) {
+ CompactionPri e1, e2;
+ TestParseAndCompareOption(config_options,
+ OptionTypeInfo(0, OptionType::kCompactionPri),
+ "CompactionPri", iter.first, &e1, &e2);
+ ASSERT_EQ(e1, iter.second);
+ }
+ for (auto iter : OptionsHelper::compression_type_string_map) {
+ CompressionType e1, e2;
+ TestParseAndCompareOption(config_options,
+ OptionTypeInfo(0, OptionType::kCompressionType),
+ "CompressionType", iter.first, &e1, &e2);
+ ASSERT_EQ(e1, iter.second);
+ }
+ for (auto iter : OptionsHelper::compaction_stop_style_string_map) {
+ CompactionStopStyle e1, e2;
+ TestParseAndCompareOption(
+ config_options, OptionTypeInfo(0, OptionType::kCompactionStopStyle),
+ "CompactionStopStyle", iter.first, &e1, &e2);
+ ASSERT_EQ(e1, iter.second);
+ }
+ for (auto iter : OptionsHelper::checksum_type_string_map) {
+ ChecksumType e1, e2;
+ TestParseAndCompareOption(config_options,
+ OptionTypeInfo(0, OptionType::kChecksumType),
+ "CheckSumType", iter.first, &e1, &e2);
+ ASSERT_EQ(e1, iter.second);
+ }
+ for (auto iter : OptionsHelper::encoding_type_string_map) {
+ EncodingType e1, e2;
+ TestParseAndCompareOption(config_options,
+ OptionTypeInfo(0, OptionType::kEncodingType),
+ "EncodingType", iter.first, &e1, &e2);
+ ASSERT_EQ(e1, iter.second);
+ }
+}
+
+TEST_F(OptionTypeInfoTest, TestStruct) {
+ struct Basic {
+ int i = 42;
+ std::string s = "Hello";
+ };
+
+ struct Extended {
+ int j = 11;
+ Basic b;
+ };
+
+ std::unordered_map<std::string, OptionTypeInfo> basic_type_map = {
+ {"i", {offsetof(struct Basic, i), OptionType::kInt}},
+ {"s", {offsetof(struct Basic, s), OptionType::kString}},
+ };
+ OptionTypeInfo basic_info = OptionTypeInfo::Struct(
+ "b", &basic_type_map, 0, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable);
+
+ std::unordered_map<std::string, OptionTypeInfo> extended_type_map = {
+ {"j", {offsetof(struct Extended, j), OptionType::kInt}},
+ {"b", OptionTypeInfo::Struct(
+ "b", &basic_type_map, offsetof(struct Extended, b),
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
+ {"m", OptionTypeInfo::Struct(
+ "m", &basic_type_map, offsetof(struct Extended, b),
+ OptionVerificationType::kNormal, OptionTypeFlags::kMutable)},
+ };
+ OptionTypeInfo extended_info = OptionTypeInfo::Struct(
+ "e", &extended_type_map, 0, OptionVerificationType::kNormal,
+ OptionTypeFlags::kMutable);
+ Extended e1, e2;
+ ConfigOptions config_options;
+ std::string mismatch;
+ TestParseAndCompareOption(config_options, basic_info, "b", "{i=33;s=33}",
+ &e1.b, &e2.b);
+ ASSERT_EQ(e1.b.i, 33);
+ ASSERT_EQ(e1.b.s, "33");
+
+ TestParseAndCompareOption(config_options, basic_info, "b.i", "44", &e1.b,
+ &e2.b);
+ ASSERT_EQ(e1.b.i, 44);
+
+ TestParseAndCompareOption(config_options, basic_info, "i", "55", &e1.b,
+ &e2.b);
+ ASSERT_EQ(e1.b.i, 55);
+
+ e1.b.i = 0;
+
+ ASSERT_FALSE(
+ basic_info.AreEqual(config_options, "b", &e1.b, &e2.b, &mismatch));
+ ASSERT_EQ(mismatch, "b.i");
+ mismatch.clear();
+ ASSERT_FALSE(
+ basic_info.AreEqual(config_options, "b.i", &e1.b, &e2.b, &mismatch));
+ ASSERT_EQ(mismatch, "b.i");
+ mismatch.clear();
+ ASSERT_FALSE(
+ basic_info.AreEqual(config_options, "i", &e1.b, &e2.b, &mismatch));
+ ASSERT_EQ(mismatch, "b.i");
+ mismatch.clear();
+
+ e1 = e2;
+ ASSERT_NOK(basic_info.Parse(config_options, "b", "{i=33;s=33;j=44}", &e1.b));
+ ASSERT_NOK(basic_info.Parse(config_options, "b.j", "44", &e1.b));
+ ASSERT_NOK(basic_info.Parse(config_options, "j", "44", &e1.b));
+
+ TestParseAndCompareOption(config_options, extended_info, "e",
+ "b={i=55;s=55}; j=22;", &e1, &e2);
+ ASSERT_EQ(e1.b.i, 55);
+ ASSERT_EQ(e1.j, 22);
+ ASSERT_EQ(e1.b.s, "55");
+ TestParseAndCompareOption(config_options, extended_info, "e.b",
+ "{i=66;s=66;}", &e1, &e2);
+ ASSERT_EQ(e1.b.i, 66);
+ ASSERT_EQ(e1.j, 22);
+ ASSERT_EQ(e1.b.s, "66");
+ TestParseAndCompareOption(config_options, extended_info, "e.b.i", "77", &e1,
+ &e2);
+ ASSERT_EQ(e1.b.i, 77);
+ ASSERT_EQ(e1.j, 22);
+ ASSERT_EQ(e1.b.s, "66");
+}
+
+TEST_F(OptionTypeInfoTest, TestArrayType) {
+ OptionTypeInfo array_info = OptionTypeInfo::Array<std::string, 4>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone,
+ {0, OptionType::kString});
+ std::array<std::string, 4> array1, array2;
+ std::string mismatch;
+
+ ConfigOptions config_options;
+ TestParseAndCompareOption(config_options, array_info, "v", "a:b:c:d", &array1,
+ &array2);
+
+ ASSERT_EQ(array1.size(), 4);
+ ASSERT_EQ(array1[0], "a");
+ ASSERT_EQ(array1[1], "b");
+ ASSERT_EQ(array1[2], "c");
+ ASSERT_EQ(array1[3], "d");
+ array1[3] = "e";
+ ASSERT_FALSE(
+ array_info.AreEqual(config_options, "v", &array1, &array2, &mismatch));
+ ASSERT_EQ(mismatch, "v");
+
+ // Test vectors with inner brackets
+ TestParseAndCompareOption(config_options, array_info, "v", "a:{b}:c:d",
+ &array1, &array2);
+ ASSERT_EQ(array1.size(), 4);
+ ASSERT_EQ(array1[0], "a");
+ ASSERT_EQ(array1[1], "b");
+ ASSERT_EQ(array1[2], "c");
+ ASSERT_EQ(array1[3], "d");
+
+ std::array<std::string, 3> array3, array4;
+ OptionTypeInfo bar_info = OptionTypeInfo::Array<std::string, 3>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone,
+ {0, OptionType::kString}, '|');
+ TestParseAndCompareOption(config_options, bar_info, "v", "x|y|z", &array3,
+ &array4);
+
+ // Test arrays with inner array
+ TestParseAndCompareOption(config_options, bar_info, "v",
+ "a|{b1|b2}|{c1|c2|{d1|d2}}", &array3, &array4,
+ false);
+ ASSERT_EQ(array3.size(), 3);
+ ASSERT_EQ(array3[0], "a");
+ ASSERT_EQ(array3[1], "b1|b2");
+ ASSERT_EQ(array3[2], "c1|c2|{d1|d2}");
+
+ TestParseAndCompareOption(config_options, bar_info, "v",
+ "{a1|a2}|{b1|{c1|c2}}|d1", &array3, &array4, true);
+ ASSERT_EQ(array3.size(), 3);
+ ASSERT_EQ(array3[0], "a1|a2");
+ ASSERT_EQ(array3[1], "b1|{c1|c2}");
+ ASSERT_EQ(array3[2], "d1");
+
+ // Test invalid input: less element than requested
+ auto s = bar_info.Parse(config_options, "opt_name1", "a1|a2", &array3);
+ ASSERT_TRUE(s.IsInvalidArgument());
+
+ // Test invalid input: more element than requested
+ s = bar_info.Parse(config_options, "opt_name2", "a1|b|c1|d3", &array3);
+ ASSERT_TRUE(s.IsInvalidArgument());
+}
+
+TEST_F(OptionTypeInfoTest, TestVectorType) {
+ OptionTypeInfo vec_info = OptionTypeInfo::Vector<std::string>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone,
+ {0, OptionType::kString});
+ std::vector<std::string> vec1, vec2;
+ std::string mismatch;
+
+ ConfigOptions config_options;
+ TestParseAndCompareOption(config_options, vec_info, "v", "a:b:c:d", &vec1,
+ &vec2);
+ ASSERT_EQ(vec1.size(), 4);
+ ASSERT_EQ(vec1[0], "a");
+ ASSERT_EQ(vec1[1], "b");
+ ASSERT_EQ(vec1[2], "c");
+ ASSERT_EQ(vec1[3], "d");
+ vec1[3] = "e";
+ ASSERT_FALSE(vec_info.AreEqual(config_options, "v", &vec1, &vec2, &mismatch));
+ ASSERT_EQ(mismatch, "v");
+
+ // Test vectors with inner brackets
+ TestParseAndCompareOption(config_options, vec_info, "v", "a:{b}:c:d", &vec1,
+ &vec2);
+ ASSERT_EQ(vec1.size(), 4);
+ ASSERT_EQ(vec1[0], "a");
+ ASSERT_EQ(vec1[1], "b");
+ ASSERT_EQ(vec1[2], "c");
+ ASSERT_EQ(vec1[3], "d");
+
+ OptionTypeInfo bar_info = OptionTypeInfo::Vector<std::string>(
+ 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone,
+ {0, OptionType::kString}, '|');
+ TestParseAndCompareOption(config_options, vec_info, "v", "x|y|z", &vec1,
+ &vec2);
+ // Test vectors with inner vector
+ TestParseAndCompareOption(config_options, bar_info, "v",
+ "a|{b1|b2}|{c1|c2|{d1|d2}}", &vec1, &vec2, false);
+ ASSERT_EQ(vec1.size(), 3);
+ ASSERT_EQ(vec1[0], "a");
+ ASSERT_EQ(vec1[1], "b1|b2");
+ ASSERT_EQ(vec1[2], "c1|c2|{d1|d2}");
+
+ TestParseAndCompareOption(config_options, bar_info, "v",
+ "{a1|a2}|{b1|{c1|c2}}|d1", &vec1, &vec2, true);
+ ASSERT_EQ(vec1.size(), 3);
+ ASSERT_EQ(vec1[0], "a1|a2");
+ ASSERT_EQ(vec1[1], "b1|{c1|c2}");
+ ASSERT_EQ(vec1[2], "d1");
+
+ TestParseAndCompareOption(config_options, bar_info, "v", "{a1}", &vec1, &vec2,
+ false);
+ ASSERT_EQ(vec1.size(), 1);
+ ASSERT_EQ(vec1[0], "a1");
+
+ TestParseAndCompareOption(config_options, bar_info, "v", "{a1|a2}|{b1|b2}",
+ &vec1, &vec2, true);
+ ASSERT_EQ(vec1.size(), 2);
+ ASSERT_EQ(vec1[0], "a1|a2");
+ ASSERT_EQ(vec1[1], "b1|b2");
+}
+
+TEST_F(OptionTypeInfoTest, TestStaticType) {
+ struct SimpleOptions {
+ size_t size = 0;
+ bool verify = true;
+ };
+
+ static std::unordered_map<std::string, OptionTypeInfo> type_map = {
+ {"size", {offsetof(struct SimpleOptions, size), OptionType::kSizeT}},
+ {"verify",
+ {offsetof(struct SimpleOptions, verify), OptionType::kBoolean}},
+ };
+
+ ConfigOptions config_options;
+ SimpleOptions opts, copy;
+ opts.size = 12345;
+ opts.verify = false;
+ std::string str, mismatch;
+
+ ASSERT_OK(
+ OptionTypeInfo::SerializeType(config_options, type_map, &opts, &str));
+ ASSERT_FALSE(OptionTypeInfo::TypesAreEqual(config_options, type_map, &opts,
+ &copy, &mismatch));
+ ASSERT_OK(OptionTypeInfo::ParseType(config_options, str, type_map, &copy));
+ ASSERT_TRUE(OptionTypeInfo::TypesAreEqual(config_options, type_map, &opts,
+ &copy, &mismatch));
+}
+
+class ConfigOptionsTest : public testing::Test {};
+
+TEST_F(ConfigOptionsTest, EnvFromConfigOptions) {
+ ConfigOptions config_options;
+ DBOptions db_opts;
+ Options opts;
+ Env* mem_env = NewMemEnv(Env::Default());
+ config_options.registry->AddLibrary("custom-env", RegisterCustomEnv,
+ kCustomEnvName);
+
+ config_options.env = mem_env;
+ // First test that we can get the env as expected
+ ASSERT_OK(GetDBOptionsFromString(config_options, DBOptions(), kCustomEnvProp,
+ &db_opts));
+ ASSERT_OK(
+ GetOptionsFromString(config_options, Options(), kCustomEnvProp, &opts));
+ ASSERT_NE(config_options.env, db_opts.env);
+ ASSERT_EQ(opts.env, db_opts.env);
+ Env* custom_env = db_opts.env;
+
+ // Now try a "bad" env" and check that nothing changed
+ config_options.ignore_unsupported_options = true;
+ ASSERT_OK(
+ GetDBOptionsFromString(config_options, db_opts, "env=unknown", &db_opts));
+ ASSERT_OK(GetOptionsFromString(config_options, opts, "env=unknown", &opts));
+ ASSERT_EQ(config_options.env, mem_env);
+ ASSERT_EQ(db_opts.env, custom_env);
+ ASSERT_EQ(opts.env, db_opts.env);
+
+ // Now try a "bad" env" ignoring unknown objects
+ config_options.ignore_unsupported_options = false;
+ ASSERT_NOK(
+ GetDBOptionsFromString(config_options, db_opts, "env=unknown", &db_opts));
+ ASSERT_EQ(config_options.env, mem_env);
+ ASSERT_EQ(db_opts.env, custom_env);
+ ASSERT_EQ(opts.env, db_opts.env);
+
+ delete mem_env;
+}
+TEST_F(ConfigOptionsTest, MergeOperatorFromString) {
+ ConfigOptions config_options;
+ std::shared_ptr<MergeOperator> merge_op;
+
+ ASSERT_OK(MergeOperator::CreateFromString(config_options, "put", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("put"));
+ ASSERT_STREQ(merge_op->Name(), "PutOperator");
+
+ ASSERT_OK(
+ MergeOperator::CreateFromString(config_options, "put_v1", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("PutOperator"));
+
+ ASSERT_OK(
+ MergeOperator::CreateFromString(config_options, "uint64add", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("uint64add"));
+ ASSERT_STREQ(merge_op->Name(), "UInt64AddOperator");
+
+ ASSERT_OK(MergeOperator::CreateFromString(config_options, "max", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("max"));
+ ASSERT_STREQ(merge_op->Name(), "MaxOperator");
+
+ ASSERT_OK(
+ MergeOperator::CreateFromString(config_options, "bytesxor", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("bytesxor"));
+ ASSERT_STREQ(merge_op->Name(), BytesXOROperator::kClassName());
+
+ ASSERT_OK(
+ MergeOperator::CreateFromString(config_options, "sortlist", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("sortlist"));
+ ASSERT_STREQ(merge_op->Name(), SortList::kClassName());
+
+ ASSERT_OK(MergeOperator::CreateFromString(config_options, "stringappend",
+ &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("stringappend"));
+ ASSERT_STREQ(merge_op->Name(), StringAppendOperator::kClassName());
+ auto delimiter = merge_op->GetOptions<std::string>("Delimiter");
+ ASSERT_NE(delimiter, nullptr);
+ ASSERT_EQ(*delimiter, ",");
+
+ ASSERT_OK(MergeOperator::CreateFromString(config_options, "stringappendtest",
+ &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("stringappendtest"));
+ ASSERT_STREQ(merge_op->Name(), StringAppendTESTOperator::kClassName());
+ delimiter = merge_op->GetOptions<std::string>("Delimiter");
+ ASSERT_NE(delimiter, nullptr);
+ ASSERT_EQ(*delimiter, ",");
+
+ ASSERT_OK(MergeOperator::CreateFromString(
+ config_options, "id=stringappend; delimiter=||", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("stringappend"));
+ ASSERT_STREQ(merge_op->Name(), StringAppendOperator::kClassName());
+ delimiter = merge_op->GetOptions<std::string>("Delimiter");
+ ASSERT_NE(delimiter, nullptr);
+ ASSERT_EQ(*delimiter, "||");
+
+ ASSERT_OK(MergeOperator::CreateFromString(
+ config_options, "id=stringappendtest; delimiter=&&", &merge_op));
+ ASSERT_NE(merge_op, nullptr);
+ ASSERT_TRUE(merge_op->IsInstanceOf("stringappendtest"));
+ ASSERT_STREQ(merge_op->Name(), StringAppendTESTOperator::kClassName());
+ delimiter = merge_op->GetOptions<std::string>("Delimiter");
+ ASSERT_NE(delimiter, nullptr);
+ ASSERT_EQ(*delimiter, "&&");
+
+ std::shared_ptr<MergeOperator> copy;
+ std::string mismatch;
+ std::string opts_str = merge_op->ToString(config_options);
+
+ ASSERT_OK(MergeOperator::CreateFromString(config_options, opts_str, &copy));
+ ASSERT_TRUE(merge_op->AreEquivalent(config_options, copy.get(), &mismatch));
+ ASSERT_NE(copy, nullptr);
+ delimiter = copy->GetOptions<std::string>("Delimiter");
+ ASSERT_NE(delimiter, nullptr);
+ ASSERT_EQ(*delimiter, "&&");
+}
+
+TEST_F(ConfigOptionsTest, ConfiguringOptionsDoesNotRevertRateLimiterBandwidth) {
+ // Regression test for bug where rate limiter's dynamically set bandwidth
+ // could be silently reverted when configuring an options structure with an
+ // existing `rate_limiter`.
+ Options base_options;
+ base_options.rate_limiter.reset(
+ NewGenericRateLimiter(1 << 20 /* rate_bytes_per_sec */));
+ Options copy_options(base_options);
+
+ base_options.rate_limiter->SetBytesPerSecond(2 << 20);
+ ASSERT_EQ(2 << 20, base_options.rate_limiter->GetBytesPerSecond());
+
+ ASSERT_OK(GetOptionsFromString(base_options, "", &copy_options));
+ ASSERT_EQ(2 << 20, base_options.rate_limiter->GetBytesPerSecond());
+}
+
+INSTANTIATE_TEST_CASE_P(OptionsSanityCheckTest, OptionsSanityCheckTest,
+ ::testing::Bool());
+#endif // !ROCKSDB_LITE
+
+} // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+ ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+ ::testing::InitGoogleTest(&argc, argv);
+#ifdef GFLAGS
+ ParseCommandLineFlags(&argc, &argv, true);
+#endif // GFLAGS
+ return RUN_ALL_TESTS();
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-17 06:52:47 +0000