diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 18:24:20 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-27 18:24:20 +0000 |
commit | 483eb2f56657e8e7f419ab1a4fab8dce9ade8609 (patch) | |
tree | e5d88d25d870d5dedacb6bbdbe2a966086a0a5cf /src/rocksdb/options | |
parent | Initial commit. (diff) | |
download | ceph-upstream.tar.xz ceph-upstream.zip |
Adding upstream version 14.2.21.upstream/14.2.21upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/rocksdb/options')
-rw-r--r-- | src/rocksdb/options/cf_options.cc | 228 | ||||
-rw-r--r-- | src/rocksdb/options/cf_options.h | 261 | ||||
-rw-r--r-- | src/rocksdb/options/db_options.cc | 301 | ||||
-rw-r--r-- | src/rocksdb/options/db_options.h | 109 | ||||
-rw-r--r-- | src/rocksdb/options/options.cc | 591 | ||||
-rw-r--r-- | src/rocksdb/options/options_helper.cc | 2043 | ||||
-rw-r--r-- | src/rocksdb/options/options_helper.h | 226 | ||||
-rw-r--r-- | src/rocksdb/options/options_parser.cc | 818 | ||||
-rw-r--r-- | src/rocksdb/options/options_parser.h | 145 | ||||
-rw-r--r-- | src/rocksdb/options/options_sanity_check.cc | 38 | ||||
-rw-r--r-- | src/rocksdb/options/options_sanity_check.h | 48 | ||||
-rw-r--r-- | src/rocksdb/options/options_settable_test.cc | 482 | ||||
-rw-r--r-- | src/rocksdb/options/options_test.cc | 1908 |
13 files changed, 7198 insertions, 0 deletions
diff --git a/src/rocksdb/options/cf_options.cc b/src/rocksdb/options/cf_options.cc new file mode 100644 index 00000000..6957e150 --- /dev/null +++ b/src/rocksdb/options/cf_options.cc @@ -0,0 +1,228 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "options/cf_options.h" + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include <inttypes.h> +#include <cassert> +#include <limits> +#include <string> +#include "options/db_options.h" +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "rocksdb/concurrent_task_limiter.h" + +namespace rocksdb { + +ImmutableCFOptions::ImmutableCFOptions(const Options& options) + : ImmutableCFOptions(ImmutableDBOptions(options), options) {} + +ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options, + const ColumnFamilyOptions& cf_options) + : compaction_style(cf_options.compaction_style), + compaction_pri(cf_options.compaction_pri), + user_comparator(cf_options.comparator), + internal_comparator(InternalKeyComparator(cf_options.comparator)), + merge_operator(cf_options.merge_operator.get()), + compaction_filter(cf_options.compaction_filter), + compaction_filter_factory(cf_options.compaction_filter_factory.get()), + min_write_buffer_number_to_merge( + cf_options.min_write_buffer_number_to_merge), + max_write_buffer_number_to_maintain( + cf_options.max_write_buffer_number_to_maintain), + inplace_update_support(cf_options.inplace_update_support), + inplace_callback(cf_options.inplace_callback), + info_log(db_options.info_log.get()), + statistics(db_options.statistics.get()), + rate_limiter(db_options.rate_limiter.get()), + info_log_level(db_options.info_log_level), + env(db_options.env), + allow_mmap_reads(db_options.allow_mmap_reads), + allow_mmap_writes(db_options.allow_mmap_writes), + db_paths(db_options.db_paths), + memtable_factory(cf_options.memtable_factory.get()), + table_factory(cf_options.table_factory.get()), + table_properties_collector_factories( + cf_options.table_properties_collector_factories), + advise_random_on_open(db_options.advise_random_on_open), + bloom_locality(cf_options.bloom_locality), + purge_redundant_kvs_while_flush( + cf_options.purge_redundant_kvs_while_flush), + use_fsync(db_options.use_fsync), + compression_per_level(cf_options.compression_per_level), + bottommost_compression(cf_options.bottommost_compression), + bottommost_compression_opts(cf_options.bottommost_compression_opts), + compression_opts(cf_options.compression_opts), + level_compaction_dynamic_level_bytes( + cf_options.level_compaction_dynamic_level_bytes), + access_hint_on_compaction_start( + db_options.access_hint_on_compaction_start), + new_table_reader_for_compaction_inputs( + db_options.new_table_reader_for_compaction_inputs), + num_levels(cf_options.num_levels), + optimize_filters_for_hits(cf_options.optimize_filters_for_hits), + force_consistency_checks(cf_options.force_consistency_checks), + allow_ingest_behind(db_options.allow_ingest_behind), + preserve_deletes(db_options.preserve_deletes), + listeners(db_options.listeners), + row_cache(db_options.row_cache), + max_subcompactions(db_options.max_subcompactions), + memtable_insert_with_hint_prefix_extractor( + cf_options.memtable_insert_with_hint_prefix_extractor.get()), + cf_paths(cf_options.cf_paths), + compaction_thread_limiter(cf_options.compaction_thread_limiter) {} + +// Multiple two operands. If they overflow, return op1. +uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) { + if (op1 == 0 || op2 <= 0) { + return 0; + } + if (port::kMaxUint64 / op1 < op2) { + return op1; + } + return static_cast<uint64_t>(op1 * op2); +} + +// when level_compaction_dynamic_level_bytes is true and leveled compaction +// is used, the base level is not always L1, so precomupted max_file_size can +// no longer be used. Recompute file_size_for_level from base level. +uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options, + int level, CompactionStyle compaction_style, int base_level, + bool level_compaction_dynamic_level_bytes) { + if (!level_compaction_dynamic_level_bytes || level < base_level || + compaction_style != kCompactionStyleLevel) { + assert(level >= 0); + assert(level < (int)cf_options.max_file_size.size()); + return cf_options.max_file_size[level]; + } else { + assert(level >= 0 && base_level >= 0); + assert(level - base_level < (int)cf_options.max_file_size.size()); + return cf_options.max_file_size[level - base_level]; + } +} + +void MutableCFOptions::RefreshDerivedOptions(int num_levels, + CompactionStyle compaction_style) { + max_file_size.resize(num_levels); + for (int i = 0; i < num_levels; ++i) { + if (i == 0 && compaction_style == kCompactionStyleUniversal) { + max_file_size[i] = ULLONG_MAX; + } else if (i > 1) { + max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1], + target_file_size_multiplier); + } else { + max_file_size[i] = target_file_size_base; + } + } +} + +void MutableCFOptions::Dump(Logger* log) const { + // Memtable related options + ROCKS_LOG_INFO(log, + " write_buffer_size: %" ROCKSDB_PRIszt, + write_buffer_size); + ROCKS_LOG_INFO(log, " max_write_buffer_number: %d", + max_write_buffer_number); + ROCKS_LOG_INFO(log, + " arena_block_size: %" ROCKSDB_PRIszt, + arena_block_size); + ROCKS_LOG_INFO(log, " memtable_prefix_bloom_ratio: %f", + memtable_prefix_bloom_size_ratio); + ROCKS_LOG_INFO(log, " memtable_whole_key_filtering: %d", + memtable_whole_key_filtering); + ROCKS_LOG_INFO(log, + " memtable_huge_page_size: %" ROCKSDB_PRIszt, + memtable_huge_page_size); + ROCKS_LOG_INFO(log, + " max_successive_merges: %" ROCKSDB_PRIszt, + max_successive_merges); + ROCKS_LOG_INFO(log, + " inplace_update_num_locks: %" ROCKSDB_PRIszt, + inplace_update_num_locks); + ROCKS_LOG_INFO( + log, " prefix_extractor: %s", + prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name()); + ROCKS_LOG_INFO(log, " disable_auto_compactions: %d", + disable_auto_compactions); + ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64, + soft_pending_compaction_bytes_limit); + ROCKS_LOG_INFO(log, " hard_pending_compaction_bytes_limit: %" PRIu64, + hard_pending_compaction_bytes_limit); + ROCKS_LOG_INFO(log, " level0_file_num_compaction_trigger: %d", + level0_file_num_compaction_trigger); + ROCKS_LOG_INFO(log, " level0_slowdown_writes_trigger: %d", + level0_slowdown_writes_trigger); + ROCKS_LOG_INFO(log, " level0_stop_writes_trigger: %d", + level0_stop_writes_trigger); + ROCKS_LOG_INFO(log, " max_compaction_bytes: %" PRIu64, + max_compaction_bytes); + ROCKS_LOG_INFO(log, " target_file_size_base: %" PRIu64, + target_file_size_base); + ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d", + target_file_size_multiplier); + ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64, + max_bytes_for_level_base); + ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f", + max_bytes_for_level_multiplier); + ROCKS_LOG_INFO(log, " ttl: %" PRIu64, + ttl); + std::string result; + char buf[10]; + for (const auto m : max_bytes_for_level_multiplier_additional) { + snprintf(buf, sizeof(buf), "%d, ", m); + result += buf; + } + if (result.size() >= 2) { + result.resize(result.size() - 2); + } else { + result = ""; + } + + ROCKS_LOG_INFO(log, "max_bytes_for_level_multiplier_additional: %s", + result.c_str()); + ROCKS_LOG_INFO(log, " max_sequential_skip_in_iterations: %" PRIu64, + max_sequential_skip_in_iterations); + ROCKS_LOG_INFO(log, " paranoid_file_checks: %d", + paranoid_file_checks); + ROCKS_LOG_INFO(log, " report_bg_io_stats: %d", + report_bg_io_stats); + ROCKS_LOG_INFO(log, " compression: %d", + static_cast<int>(compression)); + + // Universal Compaction Options + ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d", + compaction_options_universal.size_ratio); + ROCKS_LOG_INFO(log, "compaction_options_universal.min_merge_width : %d", + compaction_options_universal.min_merge_width); + ROCKS_LOG_INFO(log, "compaction_options_universal.max_merge_width : %d", + compaction_options_universal.max_merge_width); + ROCKS_LOG_INFO( + log, "compaction_options_universal.max_size_amplification_percent : %d", + compaction_options_universal.max_size_amplification_percent); + ROCKS_LOG_INFO(log, + "compaction_options_universal.compression_size_percent : %d", + compaction_options_universal.compression_size_percent); + ROCKS_LOG_INFO(log, "compaction_options_universal.stop_style : %d", + compaction_options_universal.stop_style); + ROCKS_LOG_INFO( + log, "compaction_options_universal.allow_trivial_move : %d", + static_cast<int>(compaction_options_universal.allow_trivial_move)); + + // FIFO Compaction Options + ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64, + compaction_options_fifo.max_table_files_size); + ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d", + compaction_options_fifo.allow_compaction); +} + +MutableCFOptions::MutableCFOptions(const Options& options) + : MutableCFOptions(ColumnFamilyOptions(options)) {} + +} // namespace rocksdb diff --git a/src/rocksdb/options/cf_options.h b/src/rocksdb/options/cf_options.h new file mode 100644 index 00000000..fed144e4 --- /dev/null +++ b/src/rocksdb/options/cf_options.h @@ -0,0 +1,261 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include <string> +#include <vector> + +#include "db/dbformat.h" +#include "options/db_options.h" +#include "rocksdb/options.h" +#include "util/compression.h" + +namespace rocksdb { + +// ImmutableCFOptions is a data struct used by RocksDB internal. It contains a +// subset of Options that should not be changed during the entire lifetime +// of DB. Raw pointers defined in this struct do not have ownership to the data +// they point to. Options contains std::shared_ptr to these data. +struct ImmutableCFOptions { + ImmutableCFOptions(); + explicit ImmutableCFOptions(const Options& options); + + ImmutableCFOptions(const ImmutableDBOptions& db_options, + const ColumnFamilyOptions& cf_options); + + CompactionStyle compaction_style; + + CompactionPri compaction_pri; + + const Comparator* user_comparator; + InternalKeyComparator internal_comparator; + + MergeOperator* merge_operator; + + const CompactionFilter* compaction_filter; + + CompactionFilterFactory* compaction_filter_factory; + + int min_write_buffer_number_to_merge; + + int max_write_buffer_number_to_maintain; + + bool inplace_update_support; + + UpdateStatus (*inplace_callback)(char* existing_value, + uint32_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + + Logger* info_log; + + Statistics* statistics; + + RateLimiter* rate_limiter; + + InfoLogLevel info_log_level; + + Env* env; + + // Allow the OS to mmap file for reading sst tables. Default: false + bool allow_mmap_reads; + + // Allow the OS to mmap file for writing. Default: false + bool allow_mmap_writes; + + std::vector<DbPath> db_paths; + + MemTableRepFactory* memtable_factory; + + TableFactory* table_factory; + + Options::TablePropertiesCollectorFactories + table_properties_collector_factories; + + bool advise_random_on_open; + + // This options is required by PlainTableReader. May need to move it + // to PlainTableOptions just like bloom_bits_per_key + uint32_t bloom_locality; + + bool purge_redundant_kvs_while_flush; + + bool use_fsync; + + std::vector<CompressionType> compression_per_level; + + CompressionType bottommost_compression; + + CompressionOptions bottommost_compression_opts; + + CompressionOptions compression_opts; + + bool level_compaction_dynamic_level_bytes; + + Options::AccessHint access_hint_on_compaction_start; + + bool new_table_reader_for_compaction_inputs; + + int num_levels; + + bool optimize_filters_for_hits; + + bool force_consistency_checks; + + bool allow_ingest_behind; + + bool preserve_deletes; + + // A vector of EventListeners which callback functions will be called + // when specific RocksDB event happens. + std::vector<std::shared_ptr<EventListener>> listeners; + + std::shared_ptr<Cache> row_cache; + + uint32_t max_subcompactions; + + const SliceTransform* memtable_insert_with_hint_prefix_extractor; + + std::vector<DbPath> cf_paths; + + std::shared_ptr<ConcurrentTaskLimiter> compaction_thread_limiter; +}; + +struct MutableCFOptions { + explicit MutableCFOptions(const ColumnFamilyOptions& options) + : write_buffer_size(options.write_buffer_size), + max_write_buffer_number(options.max_write_buffer_number), + arena_block_size(options.arena_block_size), + memtable_prefix_bloom_size_ratio( + options.memtable_prefix_bloom_size_ratio), + memtable_whole_key_filtering(options.memtable_whole_key_filtering), + memtable_huge_page_size(options.memtable_huge_page_size), + max_successive_merges(options.max_successive_merges), + inplace_update_num_locks(options.inplace_update_num_locks), + prefix_extractor(options.prefix_extractor), + disable_auto_compactions(options.disable_auto_compactions), + soft_pending_compaction_bytes_limit( + options.soft_pending_compaction_bytes_limit), + hard_pending_compaction_bytes_limit( + options.hard_pending_compaction_bytes_limit), + level0_file_num_compaction_trigger( + options.level0_file_num_compaction_trigger), + level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), + level0_stop_writes_trigger(options.level0_stop_writes_trigger), + max_compaction_bytes(options.max_compaction_bytes), + target_file_size_base(options.target_file_size_base), + target_file_size_multiplier(options.target_file_size_multiplier), + max_bytes_for_level_base(options.max_bytes_for_level_base), + max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), + ttl(options.ttl), + max_bytes_for_level_multiplier_additional( + options.max_bytes_for_level_multiplier_additional), + compaction_options_fifo(options.compaction_options_fifo), + compaction_options_universal(options.compaction_options_universal), + max_sequential_skip_in_iterations( + options.max_sequential_skip_in_iterations), + paranoid_file_checks(options.paranoid_file_checks), + report_bg_io_stats(options.report_bg_io_stats), + compression(options.compression), + sample_for_compression(options.sample_for_compression) { + RefreshDerivedOptions(options.num_levels, options.compaction_style); + } + + MutableCFOptions() + : write_buffer_size(0), + max_write_buffer_number(0), + arena_block_size(0), + memtable_prefix_bloom_size_ratio(0), + memtable_whole_key_filtering(false), + memtable_huge_page_size(0), + max_successive_merges(0), + inplace_update_num_locks(0), + prefix_extractor(nullptr), + disable_auto_compactions(false), + soft_pending_compaction_bytes_limit(0), + hard_pending_compaction_bytes_limit(0), + level0_file_num_compaction_trigger(0), + level0_slowdown_writes_trigger(0), + level0_stop_writes_trigger(0), + max_compaction_bytes(0), + target_file_size_base(0), + target_file_size_multiplier(0), + max_bytes_for_level_base(0), + max_bytes_for_level_multiplier(0), + ttl(0), + compaction_options_fifo(), + max_sequential_skip_in_iterations(0), + paranoid_file_checks(false), + report_bg_io_stats(false), + compression(Snappy_Supported() ? kSnappyCompression : kNoCompression), + sample_for_compression(0) {} + + explicit MutableCFOptions(const Options& options); + + // Must be called after any change to MutableCFOptions + void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style); + + void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) { + RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style); + } + + int MaxBytesMultiplerAdditional(int level) const { + if (level >= + static_cast<int>(max_bytes_for_level_multiplier_additional.size())) { + return 1; + } + return max_bytes_for_level_multiplier_additional[level]; + } + + void Dump(Logger* log) const; + + // Memtable related options + size_t write_buffer_size; + int max_write_buffer_number; + size_t arena_block_size; + double memtable_prefix_bloom_size_ratio; + bool memtable_whole_key_filtering; + size_t memtable_huge_page_size; + size_t max_successive_merges; + size_t inplace_update_num_locks; + std::shared_ptr<const SliceTransform> prefix_extractor; + + // Compaction related options + bool disable_auto_compactions; + uint64_t soft_pending_compaction_bytes_limit; + uint64_t hard_pending_compaction_bytes_limit; + int level0_file_num_compaction_trigger; + int level0_slowdown_writes_trigger; + int level0_stop_writes_trigger; + uint64_t max_compaction_bytes; + uint64_t target_file_size_base; + int target_file_size_multiplier; + uint64_t max_bytes_for_level_base; + double max_bytes_for_level_multiplier; + uint64_t ttl; + std::vector<int> max_bytes_for_level_multiplier_additional; + CompactionOptionsFIFO compaction_options_fifo; + CompactionOptionsUniversal compaction_options_universal; + + // Misc options + uint64_t max_sequential_skip_in_iterations; + bool paranoid_file_checks; + bool report_bg_io_stats; + CompressionType compression; + uint64_t sample_for_compression; + + // Derived options + // Per-level target file size. + std::vector<uint64_t> max_file_size; +}; + +uint64_t MultiplyCheckOverflow(uint64_t op1, double op2); + +// Get the max file size in a given level. +uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options, + int level, CompactionStyle compaction_style, int base_level = 1, + bool level_compaction_dynamic_level_bytes = false); +} // namespace rocksdb diff --git a/src/rocksdb/options/db_options.cc b/src/rocksdb/options/db_options.cc new file mode 100644 index 00000000..f24705cb --- /dev/null +++ b/src/rocksdb/options/db_options.cc @@ -0,0 +1,301 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "options/db_options.h" + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include <inttypes.h> + +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/env.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/wal_filter.h" +#include "util/logging.h" + +namespace rocksdb { + +ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {} + +ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) + : create_if_missing(options.create_if_missing), + create_missing_column_families(options.create_missing_column_families), + error_if_exists(options.error_if_exists), + paranoid_checks(options.paranoid_checks), + env(options.env), + rate_limiter(options.rate_limiter), + sst_file_manager(options.sst_file_manager), + info_log(options.info_log), + info_log_level(options.info_log_level), + max_file_opening_threads(options.max_file_opening_threads), + statistics(options.statistics), + use_fsync(options.use_fsync), + db_paths(options.db_paths), + db_log_dir(options.db_log_dir), + wal_dir(options.wal_dir), + max_subcompactions(options.max_subcompactions), + max_background_flushes(options.max_background_flushes), + max_log_file_size(options.max_log_file_size), + log_file_time_to_roll(options.log_file_time_to_roll), + keep_log_file_num(options.keep_log_file_num), + recycle_log_file_num(options.recycle_log_file_num), + max_manifest_file_size(options.max_manifest_file_size), + table_cache_numshardbits(options.table_cache_numshardbits), + wal_ttl_seconds(options.WAL_ttl_seconds), + wal_size_limit_mb(options.WAL_size_limit_MB), + manifest_preallocation_size(options.manifest_preallocation_size), + allow_mmap_reads(options.allow_mmap_reads), + allow_mmap_writes(options.allow_mmap_writes), + use_direct_reads(options.use_direct_reads), + use_direct_io_for_flush_and_compaction( + options.use_direct_io_for_flush_and_compaction), + allow_fallocate(options.allow_fallocate), + is_fd_close_on_exec(options.is_fd_close_on_exec), + advise_random_on_open(options.advise_random_on_open), + db_write_buffer_size(options.db_write_buffer_size), + write_buffer_manager(options.write_buffer_manager), + access_hint_on_compaction_start(options.access_hint_on_compaction_start), + new_table_reader_for_compaction_inputs( + options.new_table_reader_for_compaction_inputs), + random_access_max_buffer_size(options.random_access_max_buffer_size), + use_adaptive_mutex(options.use_adaptive_mutex), + listeners(options.listeners), + enable_thread_tracking(options.enable_thread_tracking), + enable_pipelined_write(options.enable_pipelined_write), + allow_concurrent_memtable_write(options.allow_concurrent_memtable_write), + enable_write_thread_adaptive_yield( + options.enable_write_thread_adaptive_yield), + write_thread_max_yield_usec(options.write_thread_max_yield_usec), + write_thread_slow_yield_usec(options.write_thread_slow_yield_usec), + skip_stats_update_on_db_open(options.skip_stats_update_on_db_open), + wal_recovery_mode(options.wal_recovery_mode), + allow_2pc(options.allow_2pc), + row_cache(options.row_cache), +#ifndef ROCKSDB_LITE + wal_filter(options.wal_filter), +#endif // ROCKSDB_LITE + fail_if_options_file_error(options.fail_if_options_file_error), + dump_malloc_stats(options.dump_malloc_stats), + avoid_flush_during_recovery(options.avoid_flush_during_recovery), + allow_ingest_behind(options.allow_ingest_behind), + preserve_deletes(options.preserve_deletes), + two_write_queues(options.two_write_queues), + manual_wal_flush(options.manual_wal_flush), + atomic_flush(options.atomic_flush), + avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io) { +} + +void ImmutableDBOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.error_if_exists: %d", + error_if_exists); + ROCKS_LOG_HEADER(log, " Options.create_if_missing: %d", + create_if_missing); + ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d", + paranoid_checks); + ROCKS_LOG_HEADER(log, " Options.env: %p", + env); + ROCKS_LOG_HEADER(log, " Options.info_log: %p", + info_log.get()); + ROCKS_LOG_HEADER(log, " Options.max_file_opening_threads: %d", + max_file_opening_threads); + ROCKS_LOG_HEADER(log, " Options.statistics: %p", + statistics.get()); + ROCKS_LOG_HEADER(log, " Options.use_fsync: %d", + use_fsync); + ROCKS_LOG_HEADER( + log, " Options.max_log_file_size: %" ROCKSDB_PRIszt, + max_log_file_size); + ROCKS_LOG_HEADER(log, + " Options.max_manifest_file_size: %" PRIu64, + max_manifest_file_size); + ROCKS_LOG_HEADER( + log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt, + log_file_time_to_roll); + ROCKS_LOG_HEADER( + log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt, + keep_log_file_num); + ROCKS_LOG_HEADER( + log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt, + recycle_log_file_num); + ROCKS_LOG_HEADER(log, " Options.allow_fallocate: %d", + allow_fallocate); + ROCKS_LOG_HEADER(log, " Options.allow_mmap_reads: %d", + allow_mmap_reads); + ROCKS_LOG_HEADER(log, " Options.allow_mmap_writes: %d", + allow_mmap_writes); + ROCKS_LOG_HEADER(log, " Options.use_direct_reads: %d", + use_direct_reads); + ROCKS_LOG_HEADER(log, + " " + "Options.use_direct_io_for_flush_and_compaction: %d", + use_direct_io_for_flush_and_compaction); + ROCKS_LOG_HEADER(log, " Options.create_missing_column_families: %d", + create_missing_column_families); + ROCKS_LOG_HEADER(log, " Options.db_log_dir: %s", + db_log_dir.c_str()); + ROCKS_LOG_HEADER(log, " Options.wal_dir: %s", + wal_dir.c_str()); + ROCKS_LOG_HEADER(log, " Options.table_cache_numshardbits: %d", + table_cache_numshardbits); + ROCKS_LOG_HEADER(log, + " Options.max_subcompactions: %" PRIu32, + max_subcompactions); + ROCKS_LOG_HEADER(log, " Options.max_background_flushes: %d", + max_background_flushes); + ROCKS_LOG_HEADER(log, + " Options.WAL_ttl_seconds: %" PRIu64, + wal_ttl_seconds); + ROCKS_LOG_HEADER(log, + " Options.WAL_size_limit_MB: %" PRIu64, + wal_size_limit_mb); + ROCKS_LOG_HEADER( + log, " Options.manifest_preallocation_size: %" ROCKSDB_PRIszt, + manifest_preallocation_size); + ROCKS_LOG_HEADER(log, " Options.is_fd_close_on_exec: %d", + is_fd_close_on_exec); + ROCKS_LOG_HEADER(log, " Options.advise_random_on_open: %d", + advise_random_on_open); + ROCKS_LOG_HEADER( + log, " Options.db_write_buffer_size: %" ROCKSDB_PRIszt, + db_write_buffer_size); + ROCKS_LOG_HEADER(log, " Options.write_buffer_manager: %p", + write_buffer_manager.get()); + ROCKS_LOG_HEADER(log, " Options.access_hint_on_compaction_start: %d", + static_cast<int>(access_hint_on_compaction_start)); + ROCKS_LOG_HEADER(log, " Options.new_table_reader_for_compaction_inputs: %d", + new_table_reader_for_compaction_inputs); + ROCKS_LOG_HEADER( + log, " Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt, + random_access_max_buffer_size); + ROCKS_LOG_HEADER(log, " Options.use_adaptive_mutex: %d", + use_adaptive_mutex); + ROCKS_LOG_HEADER(log, " Options.rate_limiter: %p", + rate_limiter.get()); + Header( + log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64, + sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0); + ROCKS_LOG_HEADER(log, " Options.wal_recovery_mode: %d", + static_cast<int>(wal_recovery_mode)); + ROCKS_LOG_HEADER(log, " Options.enable_thread_tracking: %d", + enable_thread_tracking); + ROCKS_LOG_HEADER(log, " Options.enable_pipelined_write: %d", + enable_pipelined_write); + ROCKS_LOG_HEADER(log, " Options.allow_concurrent_memtable_write: %d", + allow_concurrent_memtable_write); + ROCKS_LOG_HEADER(log, " Options.enable_write_thread_adaptive_yield: %d", + enable_write_thread_adaptive_yield); + ROCKS_LOG_HEADER(log, + " Options.write_thread_max_yield_usec: %" PRIu64, + write_thread_max_yield_usec); + ROCKS_LOG_HEADER(log, + " Options.write_thread_slow_yield_usec: %" PRIu64, + write_thread_slow_yield_usec); + if (row_cache) { + ROCKS_LOG_HEADER( + log, + " Options.row_cache: %" ROCKSDB_PRIszt, + row_cache->GetCapacity()); + } else { + ROCKS_LOG_HEADER(log, + " Options.row_cache: None"); + } +#ifndef ROCKSDB_LITE + ROCKS_LOG_HEADER(log, " Options.wal_filter: %s", + wal_filter ? wal_filter->Name() : "None"); +#endif // ROCKDB_LITE + + ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_recovery: %d", + avoid_flush_during_recovery); + ROCKS_LOG_HEADER(log, " Options.allow_ingest_behind: %d", + allow_ingest_behind); + ROCKS_LOG_HEADER(log, " Options.preserve_deletes: %d", + preserve_deletes); + ROCKS_LOG_HEADER(log, " Options.two_write_queues: %d", + two_write_queues); + ROCKS_LOG_HEADER(log, " Options.manual_wal_flush: %d", + manual_wal_flush); + ROCKS_LOG_HEADER(log, " Options.atomic_flush: %d", atomic_flush); + ROCKS_LOG_HEADER(log, + " Options.avoid_unnecessary_blocking_io: %d", + avoid_unnecessary_blocking_io); +} + +MutableDBOptions::MutableDBOptions() + : max_background_jobs(2), + base_background_compactions(-1), + max_background_compactions(-1), + avoid_flush_during_shutdown(false), + writable_file_max_buffer_size(1024 * 1024), + delayed_write_rate(2 * 1024U * 1024U), + max_total_wal_size(0), + delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000), + stats_dump_period_sec(600), + stats_persist_period_sec(600), + stats_history_buffer_size(1024 * 1024), + max_open_files(-1), + bytes_per_sync(0), + wal_bytes_per_sync(0), + compaction_readahead_size(0) {} + +MutableDBOptions::MutableDBOptions(const DBOptions& options) + : max_background_jobs(options.max_background_jobs), + base_background_compactions(options.base_background_compactions), + max_background_compactions(options.max_background_compactions), + avoid_flush_during_shutdown(options.avoid_flush_during_shutdown), + writable_file_max_buffer_size(options.writable_file_max_buffer_size), + delayed_write_rate(options.delayed_write_rate), + max_total_wal_size(options.max_total_wal_size), + delete_obsolete_files_period_micros( + options.delete_obsolete_files_period_micros), + stats_dump_period_sec(options.stats_dump_period_sec), + stats_persist_period_sec(options.stats_persist_period_sec), + stats_history_buffer_size(options.stats_history_buffer_size), + max_open_files(options.max_open_files), + bytes_per_sync(options.bytes_per_sync), + wal_bytes_per_sync(options.wal_bytes_per_sync), + compaction_readahead_size(options.compaction_readahead_size) {} + +void MutableDBOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.max_background_jobs: %d", + max_background_jobs); + ROCKS_LOG_HEADER(log, " Options.max_background_compactions: %d", + max_background_compactions); + ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_shutdown: %d", + avoid_flush_during_shutdown); + ROCKS_LOG_HEADER( + log, " Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt, + writable_file_max_buffer_size); + ROCKS_LOG_HEADER(log, " Options.delayed_write_rate : %" PRIu64, + delayed_write_rate); + ROCKS_LOG_HEADER(log, " Options.max_total_wal_size: %" PRIu64, + max_total_wal_size); + ROCKS_LOG_HEADER( + log, " Options.delete_obsolete_files_period_micros: %" PRIu64, + delete_obsolete_files_period_micros); + ROCKS_LOG_HEADER(log, " Options.stats_dump_period_sec: %u", + stats_dump_period_sec); + ROCKS_LOG_HEADER(log, " Options.stats_persist_period_sec: %d", + stats_persist_period_sec); + ROCKS_LOG_HEADER( + log, + " Options.stats_history_buffer_size: %" ROCKSDB_PRIszt, + stats_history_buffer_size); + ROCKS_LOG_HEADER(log, " Options.max_open_files: %d", + max_open_files); + ROCKS_LOG_HEADER(log, + " Options.bytes_per_sync: %" PRIu64, + bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.wal_bytes_per_sync: %" PRIu64, + wal_bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.compaction_readahead_size: %" ROCKSDB_PRIszt, + compaction_readahead_size); +} + +} // namespace rocksdb diff --git a/src/rocksdb/options/db_options.h b/src/rocksdb/options/db_options.h new file mode 100644 index 00000000..283cf7d3 --- /dev/null +++ b/src/rocksdb/options/db_options.h @@ -0,0 +1,109 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include <string> +#include <vector> + +#include "rocksdb/options.h" + +namespace rocksdb { + +struct ImmutableDBOptions { + ImmutableDBOptions(); + explicit ImmutableDBOptions(const DBOptions& options); + + void Dump(Logger* log) const; + + bool create_if_missing; + bool create_missing_column_families; + bool error_if_exists; + bool paranoid_checks; + Env* env; + std::shared_ptr<RateLimiter> rate_limiter; + std::shared_ptr<SstFileManager> sst_file_manager; + std::shared_ptr<Logger> info_log; + InfoLogLevel info_log_level; + int max_file_opening_threads; + std::shared_ptr<Statistics> statistics; + bool use_fsync; + std::vector<DbPath> db_paths; + std::string db_log_dir; + std::string wal_dir; + uint32_t max_subcompactions; + int max_background_flushes; + size_t max_log_file_size; + size_t log_file_time_to_roll; + size_t keep_log_file_num; + size_t recycle_log_file_num; + uint64_t max_manifest_file_size; + int table_cache_numshardbits; + uint64_t wal_ttl_seconds; + uint64_t wal_size_limit_mb; + size_t manifest_preallocation_size; + bool allow_mmap_reads; + bool allow_mmap_writes; + bool use_direct_reads; + bool use_direct_io_for_flush_and_compaction; + bool allow_fallocate; + bool is_fd_close_on_exec; + bool advise_random_on_open; + size_t db_write_buffer_size; + std::shared_ptr<WriteBufferManager> write_buffer_manager; + DBOptions::AccessHint access_hint_on_compaction_start; + bool new_table_reader_for_compaction_inputs; + size_t random_access_max_buffer_size; + bool use_adaptive_mutex; + std::vector<std::shared_ptr<EventListener>> listeners; + bool enable_thread_tracking; + bool enable_pipelined_write; + bool allow_concurrent_memtable_write; + bool enable_write_thread_adaptive_yield; + uint64_t write_thread_max_yield_usec; + uint64_t write_thread_slow_yield_usec; + bool skip_stats_update_on_db_open; + WALRecoveryMode wal_recovery_mode; + bool allow_2pc; + std::shared_ptr<Cache> row_cache; +#ifndef ROCKSDB_LITE + WalFilter* wal_filter; +#endif // ROCKSDB_LITE + bool fail_if_options_file_error; + bool dump_malloc_stats; + bool avoid_flush_during_recovery; + bool allow_ingest_behind; + bool preserve_deletes; + bool two_write_queues; + bool manual_wal_flush; + bool atomic_flush; + bool avoid_unnecessary_blocking_io; +}; + +struct MutableDBOptions { + MutableDBOptions(); + explicit MutableDBOptions(const MutableDBOptions& options) = default; + explicit MutableDBOptions(const DBOptions& options); + + void Dump(Logger* log) const; + + int max_background_jobs; + int base_background_compactions; + int max_background_compactions; + bool avoid_flush_during_shutdown; + size_t writable_file_max_buffer_size; + uint64_t delayed_write_rate; + uint64_t max_total_wal_size; + uint64_t delete_obsolete_files_period_micros; + unsigned int stats_dump_period_sec; + unsigned int stats_persist_period_sec; + size_t stats_history_buffer_size; + int max_open_files; + uint64_t bytes_per_sync; + uint64_t wal_bytes_per_sync; + size_t compaction_readahead_size; +}; + +} // namespace rocksdb diff --git a/src/rocksdb/options/options.cc b/src/rocksdb/options/options.cc new file mode 100644 index 00000000..2c995458 --- /dev/null +++ b/src/rocksdb/options/options.cc @@ -0,0 +1,591 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/options.h" + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include <inttypes.h> +#include <limits> + +#include "monitoring/statistics.h" +#include "options/db_options.h" +#include "options/options_helper.h" +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/comparator.h" +#include "rocksdb/env.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/table.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/wal_filter.h" +#include "table/block_based_table_factory.h" +#include "util/compression.h" + +namespace rocksdb { + +AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions() { + assert(memtable_factory.get() != nullptr); +} + +AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options) + : max_write_buffer_number(options.max_write_buffer_number), + min_write_buffer_number_to_merge( + options.min_write_buffer_number_to_merge), + max_write_buffer_number_to_maintain( + options.max_write_buffer_number_to_maintain), + inplace_update_support(options.inplace_update_support), + inplace_update_num_locks(options.inplace_update_num_locks), + inplace_callback(options.inplace_callback), + memtable_prefix_bloom_size_ratio( + options.memtable_prefix_bloom_size_ratio), + memtable_whole_key_filtering(options.memtable_whole_key_filtering), + memtable_huge_page_size(options.memtable_huge_page_size), + memtable_insert_with_hint_prefix_extractor( + options.memtable_insert_with_hint_prefix_extractor), + bloom_locality(options.bloom_locality), + arena_block_size(options.arena_block_size), + compression_per_level(options.compression_per_level), + num_levels(options.num_levels), + level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), + level0_stop_writes_trigger(options.level0_stop_writes_trigger), + target_file_size_base(options.target_file_size_base), + target_file_size_multiplier(options.target_file_size_multiplier), + level_compaction_dynamic_level_bytes( + options.level_compaction_dynamic_level_bytes), + max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), + max_bytes_for_level_multiplier_additional( + options.max_bytes_for_level_multiplier_additional), + max_compaction_bytes(options.max_compaction_bytes), + soft_pending_compaction_bytes_limit( + options.soft_pending_compaction_bytes_limit), + hard_pending_compaction_bytes_limit( + options.hard_pending_compaction_bytes_limit), + compaction_style(options.compaction_style), + compaction_pri(options.compaction_pri), + compaction_options_universal(options.compaction_options_universal), + compaction_options_fifo(options.compaction_options_fifo), + max_sequential_skip_in_iterations( + options.max_sequential_skip_in_iterations), + memtable_factory(options.memtable_factory), + table_properties_collector_factories( + options.table_properties_collector_factories), + max_successive_merges(options.max_successive_merges), + optimize_filters_for_hits(options.optimize_filters_for_hits), + paranoid_file_checks(options.paranoid_file_checks), + force_consistency_checks(options.force_consistency_checks), + report_bg_io_stats(options.report_bg_io_stats), + ttl(options.ttl), + sample_for_compression(options.sample_for_compression) { + assert(memtable_factory.get() != nullptr); + if (max_bytes_for_level_multiplier_additional.size() < + static_cast<unsigned int>(num_levels)) { + max_bytes_for_level_multiplier_additional.resize(num_levels, 1); + } +} + +ColumnFamilyOptions::ColumnFamilyOptions() + : compression(Snappy_Supported() ? kSnappyCompression : kNoCompression), + table_factory( + std::shared_ptr<TableFactory>(new BlockBasedTableFactory())) {} + +ColumnFamilyOptions::ColumnFamilyOptions(const Options& options) + : ColumnFamilyOptions(*static_cast<const ColumnFamilyOptions*>(&options)) {} + +DBOptions::DBOptions() {} +DBOptions::DBOptions(const Options& options) + : DBOptions(*static_cast<const DBOptions*>(&options)) {} + +void DBOptions::Dump(Logger* log) const { + ImmutableDBOptions(*this).Dump(log); + MutableDBOptions(*this).Dump(log); +} // DBOptions::Dump + +void ColumnFamilyOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.comparator: %s", + comparator->Name()); + ROCKS_LOG_HEADER(log, " Options.merge_operator: %s", + merge_operator ? merge_operator->Name() : "None"); + ROCKS_LOG_HEADER(log, " Options.compaction_filter: %s", + compaction_filter ? compaction_filter->Name() : "None"); + ROCKS_LOG_HEADER( + log, " Options.compaction_filter_factory: %s", + compaction_filter_factory ? compaction_filter_factory->Name() : "None"); + ROCKS_LOG_HEADER(log, " Options.memtable_factory: %s", + memtable_factory->Name()); + ROCKS_LOG_HEADER(log, " Options.table_factory: %s", + table_factory->Name()); + ROCKS_LOG_HEADER(log, " table_factory options: %s", + table_factory->GetPrintableTableOptions().c_str()); + ROCKS_LOG_HEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt, + write_buffer_size); + ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number: %d", + max_write_buffer_number); + if (!compression_per_level.empty()) { + for (unsigned int i = 0; i < compression_per_level.size(); i++) { + ROCKS_LOG_HEADER( + log, " Options.compression[%d]: %s", i, + CompressionTypeToString(compression_per_level[i]).c_str()); + } + } else { + ROCKS_LOG_HEADER(log, " Options.compression: %s", + CompressionTypeToString(compression).c_str()); + } + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression: %s", + bottommost_compression == kDisableCompressionOption + ? "Disabled" + : CompressionTypeToString(bottommost_compression).c_str()); + ROCKS_LOG_HEADER( + log, " Options.prefix_extractor: %s", + prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name()); + ROCKS_LOG_HEADER(log, + " Options.memtable_insert_with_hint_prefix_extractor: %s", + memtable_insert_with_hint_prefix_extractor == nullptr + ? "nullptr" + : memtable_insert_with_hint_prefix_extractor->Name()); + ROCKS_LOG_HEADER(log, " Options.num_levels: %d", num_levels); + ROCKS_LOG_HEADER(log, " Options.min_write_buffer_number_to_merge: %d", + min_write_buffer_number_to_merge); + ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number_to_maintain: %d", + max_write_buffer_number_to_maintain); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.window_bits: %d", + bottommost_compression_opts.window_bits); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.level: %d", + bottommost_compression_opts.level); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.strategy: %d", + bottommost_compression_opts.strategy); + ROCKS_LOG_HEADER( + log, + " Options.bottommost_compression_opts.max_dict_bytes: " + "%" PRIu32, + bottommost_compression_opts.max_dict_bytes); + ROCKS_LOG_HEADER( + log, + " Options.bottommost_compression_opts.zstd_max_train_bytes: " + "%" PRIu32, + bottommost_compression_opts.zstd_max_train_bytes); + ROCKS_LOG_HEADER( + log, " Options.bottommost_compression_opts.enabled: %s", + bottommost_compression_opts.enabled ? "true" : "false"); + ROCKS_LOG_HEADER(log, " Options.compression_opts.window_bits: %d", + compression_opts.window_bits); + ROCKS_LOG_HEADER(log, " Options.compression_opts.level: %d", + compression_opts.level); + ROCKS_LOG_HEADER(log, " Options.compression_opts.strategy: %d", + compression_opts.strategy); + ROCKS_LOG_HEADER( + log, + " Options.compression_opts.max_dict_bytes: %" PRIu32, + compression_opts.max_dict_bytes); + ROCKS_LOG_HEADER(log, + " Options.compression_opts.zstd_max_train_bytes: " + "%" PRIu32, + compression_opts.zstd_max_train_bytes); + ROCKS_LOG_HEADER(log, + " Options.compression_opts.enabled: %s", + compression_opts.enabled ? "true" : "false"); + ROCKS_LOG_HEADER(log, " Options.level0_file_num_compaction_trigger: %d", + level0_file_num_compaction_trigger); + ROCKS_LOG_HEADER(log, " Options.level0_slowdown_writes_trigger: %d", + level0_slowdown_writes_trigger); + ROCKS_LOG_HEADER(log, " Options.level0_stop_writes_trigger: %d", + level0_stop_writes_trigger); + ROCKS_LOG_HEADER( + log, " Options.target_file_size_base: %" PRIu64, + target_file_size_base); + ROCKS_LOG_HEADER(log, " Options.target_file_size_multiplier: %d", + target_file_size_multiplier); + ROCKS_LOG_HEADER( + log, " Options.max_bytes_for_level_base: %" PRIu64, + max_bytes_for_level_base); + ROCKS_LOG_HEADER(log, "Options.level_compaction_dynamic_level_bytes: %d", + level_compaction_dynamic_level_bytes); + ROCKS_LOG_HEADER(log, " Options.max_bytes_for_level_multiplier: %f", + max_bytes_for_level_multiplier); + for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size(); + i++) { + ROCKS_LOG_HEADER( + log, "Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt + "]: %d", + i, max_bytes_for_level_multiplier_additional[i]); + } + ROCKS_LOG_HEADER( + log, " Options.max_sequential_skip_in_iterations: %" PRIu64, + max_sequential_skip_in_iterations); + ROCKS_LOG_HEADER( + log, " Options.max_compaction_bytes: %" PRIu64, + max_compaction_bytes); + ROCKS_LOG_HEADER( + log, + " Options.arena_block_size: %" ROCKSDB_PRIszt, + arena_block_size); + ROCKS_LOG_HEADER(log, + " Options.soft_pending_compaction_bytes_limit: %" PRIu64, + soft_pending_compaction_bytes_limit); + ROCKS_LOG_HEADER(log, + " Options.hard_pending_compaction_bytes_limit: %" PRIu64, + hard_pending_compaction_bytes_limit); + ROCKS_LOG_HEADER(log, " Options.rate_limit_delay_max_milliseconds: %u", + rate_limit_delay_max_milliseconds); + ROCKS_LOG_HEADER(log, " Options.disable_auto_compactions: %d", + disable_auto_compactions); + + const auto& it_compaction_style = + compaction_style_to_string.find(compaction_style); + std::string str_compaction_style; + if (it_compaction_style == compaction_style_to_string.end()) { + assert(false); + str_compaction_style = "unknown_" + std::to_string(compaction_style); + } else { + str_compaction_style = it_compaction_style->second; + } + ROCKS_LOG_HEADER(log, + " Options.compaction_style: %s", + str_compaction_style.c_str()); + + const auto& it_compaction_pri = + compaction_pri_to_string.find(compaction_pri); + std::string str_compaction_pri; + if (it_compaction_pri == compaction_pri_to_string.end()) { + assert(false); + str_compaction_pri = "unknown_" + std::to_string(compaction_pri); + } else { + str_compaction_pri = it_compaction_pri->second; + } + ROCKS_LOG_HEADER(log, + " Options.compaction_pri: %s", + str_compaction_pri.c_str()); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.size_ratio: %u", + compaction_options_universal.size_ratio); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.min_merge_width: %u", + compaction_options_universal.min_merge_width); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.max_merge_width: %u", + compaction_options_universal.max_merge_width); + ROCKS_LOG_HEADER( + log, + "Options.compaction_options_universal." + "max_size_amplification_percent: %u", + compaction_options_universal.max_size_amplification_percent); + ROCKS_LOG_HEADER( + log, + "Options.compaction_options_universal.compression_size_percent: %d", + compaction_options_universal.compression_size_percent); + const auto& it_compaction_stop_style = compaction_stop_style_to_string.find( + compaction_options_universal.stop_style); + std::string str_compaction_stop_style; + if (it_compaction_stop_style == compaction_stop_style_to_string.end()) { + assert(false); + str_compaction_stop_style = + "unknown_" + std::to_string(compaction_options_universal.stop_style); + } else { + str_compaction_stop_style = it_compaction_stop_style->second; + } + ROCKS_LOG_HEADER(log, + "Options.compaction_options_universal.stop_style: %s", + str_compaction_stop_style.c_str()); + ROCKS_LOG_HEADER( + log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64, + compaction_options_fifo.max_table_files_size); + ROCKS_LOG_HEADER(log, + "Options.compaction_options_fifo.allow_compaction: %d", + compaction_options_fifo.allow_compaction); + std::string collector_names; + for (const auto& collector_factory : table_properties_collector_factories) { + collector_names.append(collector_factory->Name()); + collector_names.append("; "); + } + ROCKS_LOG_HEADER( + log, " Options.table_properties_collectors: %s", + collector_names.c_str()); + ROCKS_LOG_HEADER(log, + " Options.inplace_update_support: %d", + inplace_update_support); + ROCKS_LOG_HEADER( + log, + " Options.inplace_update_num_locks: %" ROCKSDB_PRIszt, + inplace_update_num_locks); + // TODO: easier config for bloom (maybe based on avg key/value size) + ROCKS_LOG_HEADER( + log, " Options.memtable_prefix_bloom_size_ratio: %f", + memtable_prefix_bloom_size_ratio); + ROCKS_LOG_HEADER(log, + " Options.memtable_whole_key_filtering: %d", + memtable_whole_key_filtering); + + ROCKS_LOG_HEADER(log, " Options.memtable_huge_page_size: %" ROCKSDB_PRIszt, + memtable_huge_page_size); + ROCKS_LOG_HEADER(log, + " Options.bloom_locality: %d", + bloom_locality); + + ROCKS_LOG_HEADER( + log, + " Options.max_successive_merges: %" ROCKSDB_PRIszt, + max_successive_merges); + ROCKS_LOG_HEADER(log, + " Options.optimize_filters_for_hits: %d", + optimize_filters_for_hits); + ROCKS_LOG_HEADER(log, " Options.paranoid_file_checks: %d", + paranoid_file_checks); + ROCKS_LOG_HEADER(log, " Options.force_consistency_checks: %d", + force_consistency_checks); + ROCKS_LOG_HEADER(log, " Options.report_bg_io_stats: %d", + report_bg_io_stats); + ROCKS_LOG_HEADER(log, " Options.ttl: %" PRIu64, + ttl); +} // ColumnFamilyOptions::Dump + +void Options::Dump(Logger* log) const { + DBOptions::Dump(log); + ColumnFamilyOptions::Dump(log); +} // Options::Dump + +void Options::DumpCFOptions(Logger* log) const { + ColumnFamilyOptions::Dump(log); +} // Options::DumpCFOptions + +// +// The goal of this method is to create a configuration that +// allows an application to write all files into L0 and +// then do a single compaction to output all files into L1. +Options* +Options::PrepareForBulkLoad() +{ + // never slowdown ingest. + level0_file_num_compaction_trigger = (1<<30); + level0_slowdown_writes_trigger = (1<<30); + level0_stop_writes_trigger = (1<<30); + soft_pending_compaction_bytes_limit = 0; + hard_pending_compaction_bytes_limit = 0; + + // no auto compactions please. The application should issue a + // manual compaction after all data is loaded into L0. + disable_auto_compactions = true; + // A manual compaction run should pick all files in L0 in + // a single compaction run. + max_compaction_bytes = (static_cast<uint64_t>(1) << 60); + + // It is better to have only 2 levels, otherwise a manual + // compaction would compact at every possible level, thereby + // increasing the total time needed for compactions. + num_levels = 2; + + // Need to allow more write buffers to allow more parallism + // of flushes. + max_write_buffer_number = 6; + min_write_buffer_number_to_merge = 1; + + // When compaction is disabled, more parallel flush threads can + // help with write throughput. + max_background_flushes = 4; + + // Prevent a memtable flush to automatically promote files + // to L1. This is helpful so that all files that are + // input to the manual compaction are all at L0. + max_background_compactions = 2; + + // The compaction would create large files in L1. + target_file_size_base = 256 * 1024 * 1024; + return this; +} + +Options* Options::OptimizeForSmallDb() { + ColumnFamilyOptions::OptimizeForSmallDb(); + DBOptions::OptimizeForSmallDb(); + return this; +} + +Options* Options::OldDefaults(int rocksdb_major_version, + int rocksdb_minor_version) { + ColumnFamilyOptions::OldDefaults(rocksdb_major_version, + rocksdb_minor_version); + DBOptions::OldDefaults(rocksdb_major_version, rocksdb_minor_version); + return this; +} + +DBOptions* DBOptions::OldDefaults(int rocksdb_major_version, + int rocksdb_minor_version) { + if (rocksdb_major_version < 4 || + (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) { + max_file_opening_threads = 1; + table_cache_numshardbits = 4; + } + if (rocksdb_major_version < 5 || + (rocksdb_major_version == 5 && rocksdb_minor_version < 2)) { + delayed_write_rate = 2 * 1024U * 1024U; + } else if (rocksdb_major_version < 5 || + (rocksdb_major_version == 5 && rocksdb_minor_version < 6)) { + delayed_write_rate = 16 * 1024U * 1024U; + } + max_open_files = 5000; + wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OldDefaults( + int rocksdb_major_version, int rocksdb_minor_version) { + if (rocksdb_major_version < 5 || + (rocksdb_major_version == 5 && rocksdb_minor_version <= 18)) { + compaction_pri = CompactionPri::kByCompensatedSize; + } + if (rocksdb_major_version < 4 || + (rocksdb_major_version == 4 && rocksdb_minor_version < 7)) { + write_buffer_size = 4 << 20; + target_file_size_base = 2 * 1048576; + max_bytes_for_level_base = 10 * 1048576; + soft_pending_compaction_bytes_limit = 0; + hard_pending_compaction_bytes_limit = 0; + } + if (rocksdb_major_version < 5) { + level0_stop_writes_trigger = 24; + } else if (rocksdb_major_version == 5 && rocksdb_minor_version < 2) { + level0_stop_writes_trigger = 30; + } + + return this; +} + +// Optimization functions +DBOptions* DBOptions::OptimizeForSmallDb() { + max_file_opening_threads = 1; + max_open_files = 5000; + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForSmallDb() { + write_buffer_size = 2 << 20; + target_file_size_base = 2 * 1048576; + max_bytes_for_level_base = 10 * 1048576; + soft_pending_compaction_bytes_limit = 256 * 1048576; + hard_pending_compaction_bytes_limit = 1073741824ul; + return this; +} + +#ifndef ROCKSDB_LITE +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup( + uint64_t block_cache_size_mb) { + prefix_extractor.reset(NewNoopTransform()); + BlockBasedTableOptions block_based_options; + block_based_options.index_type = BlockBasedTableOptions::kHashSearch; + block_based_options.data_block_index_type = + BlockBasedTableOptions::kDataBlockBinaryAndHash; + block_based_options.data_block_hash_table_util_ratio = 0.75; + block_based_options.filter_policy.reset(NewBloomFilterPolicy(10)); + block_based_options.block_cache = + NewLRUCache(static_cast<size_t>(block_cache_size_mb * 1024 * 1024)); + table_factory.reset(new BlockBasedTableFactory(block_based_options)); + memtable_prefix_bloom_size_ratio = 0.02; + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction( + uint64_t memtable_memory_budget) { + write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4); + // merge two memtables when flushing to L0 + min_write_buffer_number_to_merge = 2; + // this means we'll use 50% extra memory in the worst case, but will reduce + // write stalls. + max_write_buffer_number = 6; + // start flushing L0->L1 as soon as possible. each file on level0 is + // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than + // memtable_memory_budget. + level0_file_num_compaction_trigger = 2; + // doesn't really matter much, but we don't want to create too many files + target_file_size_base = memtable_memory_budget / 8; + // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast + max_bytes_for_level_base = memtable_memory_budget; + + // level style compaction + compaction_style = kCompactionStyleLevel; + + // only compress levels >= 2 + compression_per_level.resize(num_levels); + for (int i = 0; i < num_levels; ++i) { + if (i < 2) { + compression_per_level[i] = kNoCompression; + } else { + compression_per_level[i] = kSnappyCompression; + } + } + return this; +} + +ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction( + uint64_t memtable_memory_budget) { + write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4); + // merge two memtables when flushing to L0 + min_write_buffer_number_to_merge = 2; + // this means we'll use 50% extra memory in the worst case, but will reduce + // write stalls. + max_write_buffer_number = 6; + // universal style compaction + compaction_style = kCompactionStyleUniversal; + compaction_options_universal.compression_size_percent = 80; + return this; +} + +DBOptions* DBOptions::IncreaseParallelism(int total_threads) { + max_background_jobs = total_threads; + env->SetBackgroundThreads(total_threads, Env::LOW); + env->SetBackgroundThreads(1, Env::HIGH); + return this; +} + +#endif // !ROCKSDB_LITE + +ReadOptions::ReadOptions() + : snapshot(nullptr), + iterate_lower_bound(nullptr), + iterate_upper_bound(nullptr), + readahead_size(0), + max_skippable_internal_keys(0), + read_tier(kReadAllTier), + verify_checksums(true), + fill_cache(true), + tailing(false), + managed(false), + total_order_seek(false), + prefix_same_as_start(false), + pin_data(false), + background_purge_on_iterator_cleanup(false), + ignore_range_deletions(false), + iter_start_seqnum(0) {} + +ReadOptions::ReadOptions(bool cksum, bool cache) + : snapshot(nullptr), + iterate_lower_bound(nullptr), + iterate_upper_bound(nullptr), + readahead_size(0), + max_skippable_internal_keys(0), + read_tier(kReadAllTier), + verify_checksums(cksum), + fill_cache(cache), + tailing(false), + managed(false), + total_order_seek(false), + prefix_same_as_start(false), + pin_data(false), + background_purge_on_iterator_cleanup(false), + ignore_range_deletions(false), + iter_start_seqnum(0) {} + +} // namespace rocksdb diff --git a/src/rocksdb/options/options_helper.cc b/src/rocksdb/options/options_helper.cc new file mode 100644 index 00000000..9facf6e9 --- /dev/null +++ b/src/rocksdb/options/options_helper.cc @@ -0,0 +1,2043 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#include "options/options_helper.h" + +#include <cassert> +#include <cctype> +#include <cstdlib> +#include <unordered_set> +#include <vector> +#include "rocksdb/cache.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/convenience.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/table.h" +#include "rocksdb/utilities/object_registry.h" +#include "table/block_based_table_factory.h" +#include "table/plain_table_factory.h" +#include "util/cast_util.h" +#include "util/string_util.h" + +namespace rocksdb { + +DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, + const MutableDBOptions& mutable_db_options) { + DBOptions options; + + options.create_if_missing = immutable_db_options.create_if_missing; + options.create_missing_column_families = + immutable_db_options.create_missing_column_families; + options.error_if_exists = immutable_db_options.error_if_exists; + options.paranoid_checks = immutable_db_options.paranoid_checks; + options.env = immutable_db_options.env; + options.rate_limiter = immutable_db_options.rate_limiter; + options.sst_file_manager = immutable_db_options.sst_file_manager; + options.info_log = immutable_db_options.info_log; + options.info_log_level = immutable_db_options.info_log_level; + options.max_open_files = mutable_db_options.max_open_files; + options.max_file_opening_threads = + immutable_db_options.max_file_opening_threads; + options.max_total_wal_size = mutable_db_options.max_total_wal_size; + options.statistics = immutable_db_options.statistics; + options.use_fsync = immutable_db_options.use_fsync; + options.db_paths = immutable_db_options.db_paths; + options.db_log_dir = immutable_db_options.db_log_dir; + options.wal_dir = immutable_db_options.wal_dir; + options.delete_obsolete_files_period_micros = + mutable_db_options.delete_obsolete_files_period_micros; + options.max_background_jobs = mutable_db_options.max_background_jobs; + options.base_background_compactions = + mutable_db_options.base_background_compactions; + options.max_background_compactions = + mutable_db_options.max_background_compactions; + options.bytes_per_sync = mutable_db_options.bytes_per_sync; + options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync; + options.max_subcompactions = immutable_db_options.max_subcompactions; + options.max_background_flushes = immutable_db_options.max_background_flushes; + options.max_log_file_size = immutable_db_options.max_log_file_size; + options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll; + options.keep_log_file_num = immutable_db_options.keep_log_file_num; + options.recycle_log_file_num = immutable_db_options.recycle_log_file_num; + options.max_manifest_file_size = immutable_db_options.max_manifest_file_size; + options.table_cache_numshardbits = + immutable_db_options.table_cache_numshardbits; + options.WAL_ttl_seconds = immutable_db_options.wal_ttl_seconds; + options.WAL_size_limit_MB = immutable_db_options.wal_size_limit_mb; + options.manifest_preallocation_size = + immutable_db_options.manifest_preallocation_size; + options.allow_mmap_reads = immutable_db_options.allow_mmap_reads; + options.allow_mmap_writes = immutable_db_options.allow_mmap_writes; + options.use_direct_reads = immutable_db_options.use_direct_reads; + options.use_direct_io_for_flush_and_compaction = + immutable_db_options.use_direct_io_for_flush_and_compaction; + options.allow_fallocate = immutable_db_options.allow_fallocate; + options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec; + options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec; + options.stats_persist_period_sec = + mutable_db_options.stats_persist_period_sec; + options.stats_history_buffer_size = + mutable_db_options.stats_history_buffer_size; + options.advise_random_on_open = immutable_db_options.advise_random_on_open; + options.db_write_buffer_size = immutable_db_options.db_write_buffer_size; + options.write_buffer_manager = immutable_db_options.write_buffer_manager; + options.access_hint_on_compaction_start = + immutable_db_options.access_hint_on_compaction_start; + options.new_table_reader_for_compaction_inputs = + immutable_db_options.new_table_reader_for_compaction_inputs; + options.compaction_readahead_size = + mutable_db_options.compaction_readahead_size; + options.random_access_max_buffer_size = + immutable_db_options.random_access_max_buffer_size; + options.writable_file_max_buffer_size = + mutable_db_options.writable_file_max_buffer_size; + options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex; + options.listeners = immutable_db_options.listeners; + options.enable_thread_tracking = immutable_db_options.enable_thread_tracking; + options.delayed_write_rate = mutable_db_options.delayed_write_rate; + options.enable_pipelined_write = immutable_db_options.enable_pipelined_write; + options.allow_concurrent_memtable_write = + immutable_db_options.allow_concurrent_memtable_write; + options.enable_write_thread_adaptive_yield = + immutable_db_options.enable_write_thread_adaptive_yield; + options.write_thread_max_yield_usec = + immutable_db_options.write_thread_max_yield_usec; + options.write_thread_slow_yield_usec = + immutable_db_options.write_thread_slow_yield_usec; + options.skip_stats_update_on_db_open = + immutable_db_options.skip_stats_update_on_db_open; + options.wal_recovery_mode = immutable_db_options.wal_recovery_mode; + options.allow_2pc = immutable_db_options.allow_2pc; + options.row_cache = immutable_db_options.row_cache; +#ifndef ROCKSDB_LITE + options.wal_filter = immutable_db_options.wal_filter; +#endif // ROCKSDB_LITE + options.fail_if_options_file_error = + immutable_db_options.fail_if_options_file_error; + options.dump_malloc_stats = immutable_db_options.dump_malloc_stats; + options.avoid_flush_during_recovery = + immutable_db_options.avoid_flush_during_recovery; + options.avoid_flush_during_shutdown = + mutable_db_options.avoid_flush_during_shutdown; + options.allow_ingest_behind = + immutable_db_options.allow_ingest_behind; + options.preserve_deletes = + immutable_db_options.preserve_deletes; + options.two_write_queues = immutable_db_options.two_write_queues; + options.manual_wal_flush = immutable_db_options.manual_wal_flush; + options.atomic_flush = immutable_db_options.atomic_flush; + options.avoid_unnecessary_blocking_io = + immutable_db_options.avoid_unnecessary_blocking_io; + + return options; +} + +ColumnFamilyOptions BuildColumnFamilyOptions( + const ColumnFamilyOptions& options, + const MutableCFOptions& mutable_cf_options) { + ColumnFamilyOptions cf_opts(options); + + // Memtable related options + cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size; + cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number; + cf_opts.arena_block_size = mutable_cf_options.arena_block_size; + cf_opts.memtable_prefix_bloom_size_ratio = + mutable_cf_options.memtable_prefix_bloom_size_ratio; + cf_opts.memtable_whole_key_filtering = + mutable_cf_options.memtable_whole_key_filtering; + cf_opts.memtable_huge_page_size = mutable_cf_options.memtable_huge_page_size; + cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges; + cf_opts.inplace_update_num_locks = + mutable_cf_options.inplace_update_num_locks; + cf_opts.prefix_extractor = mutable_cf_options.prefix_extractor; + + // Compaction related options + cf_opts.disable_auto_compactions = + mutable_cf_options.disable_auto_compactions; + cf_opts.soft_pending_compaction_bytes_limit = + mutable_cf_options.soft_pending_compaction_bytes_limit; + cf_opts.hard_pending_compaction_bytes_limit = + mutable_cf_options.hard_pending_compaction_bytes_limit; + cf_opts.level0_file_num_compaction_trigger = + mutable_cf_options.level0_file_num_compaction_trigger; + cf_opts.level0_slowdown_writes_trigger = + mutable_cf_options.level0_slowdown_writes_trigger; + cf_opts.level0_stop_writes_trigger = + mutable_cf_options.level0_stop_writes_trigger; + cf_opts.max_compaction_bytes = mutable_cf_options.max_compaction_bytes; + cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base; + cf_opts.target_file_size_multiplier = + mutable_cf_options.target_file_size_multiplier; + cf_opts.max_bytes_for_level_base = + mutable_cf_options.max_bytes_for_level_base; + cf_opts.max_bytes_for_level_multiplier = + mutable_cf_options.max_bytes_for_level_multiplier; + cf_opts.ttl = mutable_cf_options.ttl; + + cf_opts.max_bytes_for_level_multiplier_additional.clear(); + for (auto value : + mutable_cf_options.max_bytes_for_level_multiplier_additional) { + cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value); + } + + cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo; + cf_opts.compaction_options_universal = + mutable_cf_options.compaction_options_universal; + + // Misc options + cf_opts.max_sequential_skip_in_iterations = + mutable_cf_options.max_sequential_skip_in_iterations; + cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks; + cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats; + cf_opts.compression = mutable_cf_options.compression; + cf_opts.sample_for_compression = mutable_cf_options.sample_for_compression; + + cf_opts.table_factory = options.table_factory; + // TODO(yhchiang): find some way to handle the following derived options + // * max_file_size + + return cf_opts; +} + +std::map<CompactionStyle, std::string> + OptionsHelper::compaction_style_to_string = { + {kCompactionStyleLevel, "kCompactionStyleLevel"}, + {kCompactionStyleUniversal, "kCompactionStyleUniversal"}, + {kCompactionStyleFIFO, "kCompactionStyleFIFO"}, + {kCompactionStyleNone, "kCompactionStyleNone"}}; + +std::map<CompactionPri, std::string> OptionsHelper::compaction_pri_to_string = { + {kByCompensatedSize, "kByCompensatedSize"}, + {kOldestLargestSeqFirst, "kOldestLargestSeqFirst"}, + {kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"}, + {kMinOverlappingRatio, "kMinOverlappingRatio"}}; + +std::map<CompactionStopStyle, std::string> + OptionsHelper::compaction_stop_style_to_string = { + {kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"}, + {kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}}; + +std::unordered_map<std::string, ChecksumType> + OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum}, + {"kCRC32c", kCRC32c}, + {"kxxHash", kxxHash}, + {"kxxHash64", kxxHash64}}; + +std::unordered_map<std::string, CompressionType> + OptionsHelper::compression_type_string_map = { + {"kNoCompression", kNoCompression}, + {"kSnappyCompression", kSnappyCompression}, + {"kZlibCompression", kZlibCompression}, + {"kBZip2Compression", kBZip2Compression}, + {"kLZ4Compression", kLZ4Compression}, + {"kLZ4HCCompression", kLZ4HCCompression}, + {"kXpressCompression", kXpressCompression}, + {"kZSTD", kZSTD}, + {"kZSTDNotFinalCompression", kZSTDNotFinalCompression}, + {"kDisableCompressionOption", kDisableCompressionOption}}; +#ifndef ROCKSDB_LITE + +const std::string kNameComparator = "comparator"; +const std::string kNameMergeOperator = "merge_operator"; + +template <typename T> +Status GetStringFromStruct( + std::string* opt_string, const T& options, + const std::unordered_map<std::string, OptionTypeInfo> type_info, + const std::string& delimiter); + +namespace { +template <typename T> +bool ParseEnum(const std::unordered_map<std::string, T>& type_map, + const std::string& type, T* value) { + auto iter = type_map.find(type); + if (iter != type_map.end()) { + *value = iter->second; + return true; + } + return false; +} + +template <typename T> +bool SerializeEnum(const std::unordered_map<std::string, T>& type_map, + const T& type, std::string* value) { + for (const auto& pair : type_map) { + if (pair.second == type) { + *value = pair.first; + return true; + } + } + return false; +} + +bool SerializeVectorCompressionType(const std::vector<CompressionType>& types, + std::string* value) { + std::stringstream ss; + bool result; + for (size_t i = 0; i < types.size(); ++i) { + if (i > 0) { + ss << ':'; + } + std::string string_type; + result = SerializeEnum<CompressionType>(compression_type_string_map, + types[i], &string_type); + if (result == false) { + return result; + } + ss << string_type; + } + *value = ss.str(); + return true; +} + +bool ParseVectorCompressionType( + const std::string& value, + std::vector<CompressionType>* compression_per_level) { + compression_per_level->clear(); + size_t start = 0; + while (start < value.size()) { + size_t end = value.find(':', start); + bool is_ok; + CompressionType type; + if (end == std::string::npos) { + is_ok = ParseEnum<CompressionType>(compression_type_string_map, + value.substr(start), &type); + if (!is_ok) { + return false; + } + compression_per_level->emplace_back(type); + break; + } else { + is_ok = ParseEnum<CompressionType>( + compression_type_string_map, value.substr(start, end - start), &type); + if (!is_ok) { + return false; + } + compression_per_level->emplace_back(type); + start = end + 1; + } + } + return true; +} + +// This is to handle backward compatibility, where compaction_options_fifo +// could be assigned a single scalar value, say, like "23", which would be +// assigned to max_table_files_size. +bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str, + CompactionOptionsFIFO* options) { + if (opt_str.find("=") != std::string::npos) { + // New format. Go do your new parsing using ParseStructOptions. + return false; + } + + // Old format. Parse just a single uint64_t value. + options->max_table_files_size = ParseUint64(opt_str); + return true; +} + +template <typename T> +bool SerializeStruct( + const T& options, std::string* value, + std::unordered_map<std::string, OptionTypeInfo> type_info_map) { + std::string opt_str; + Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";"); + if (!s.ok()) { + return false; + } + *value = "{" + opt_str + "}"; + return true; +} + +template <typename T> +bool ParseSingleStructOption( + const std::string& opt_val_str, T* options, + std::unordered_map<std::string, OptionTypeInfo> type_info_map) { + size_t end = opt_val_str.find('='); + std::string key = opt_val_str.substr(0, end); + std::string value = opt_val_str.substr(end + 1); + auto iter = type_info_map.find(key); + if (iter == type_info_map.end()) { + return false; + } + const auto& opt_info = iter->second; + return ParseOptionHelper( + reinterpret_cast<char*>(options) + opt_info.mutable_offset, opt_info.type, + value); +} + +template <typename T> +bool ParseStructOptions( + const std::string& opt_str, T* options, + std::unordered_map<std::string, OptionTypeInfo> type_info_map) { + assert(!opt_str.empty()); + + size_t start = 0; + if (opt_str[0] == '{') { + start++; + } + while ((start != std::string::npos) && (start < opt_str.size())) { + if (opt_str[start] == '}') { + break; + } + size_t end = opt_str.find(';', start); + size_t len = (end == std::string::npos) ? end : end - start; + if (!ParseSingleStructOption(opt_str.substr(start, len), options, + type_info_map)) { + return false; + } + start = (end == std::string::npos) ? end : end + 1; + } + return true; +} +} // anonymouse namespace + +bool ParseSliceTransformHelper( + const std::string& kFixedPrefixName, const std::string& kCappedPrefixName, + const std::string& value, + std::shared_ptr<const SliceTransform>* slice_transform) { + const char* no_op_name = "rocksdb.Noop"; + size_t no_op_length = strlen(no_op_name); + auto& pe_value = value; + if (pe_value.size() > kFixedPrefixName.size() && + pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) { + int prefix_length = ParseInt(trim(value.substr(kFixedPrefixName.size()))); + slice_transform->reset(NewFixedPrefixTransform(prefix_length)); + } else if (pe_value.size() > kCappedPrefixName.size() && + pe_value.compare(0, kCappedPrefixName.size(), kCappedPrefixName) == + 0) { + int prefix_length = + ParseInt(trim(pe_value.substr(kCappedPrefixName.size()))); + slice_transform->reset(NewCappedPrefixTransform(prefix_length)); + } else if (pe_value.size() == no_op_length && + pe_value.compare(0, no_op_length, no_op_name) == 0) { + const SliceTransform* no_op_transform = NewNoopTransform(); + slice_transform->reset(no_op_transform); + } else if (value == kNullptrString) { + slice_transform->reset(); + } else { + return false; + } + + return true; +} + +bool ParseSliceTransform( + const std::string& value, + std::shared_ptr<const SliceTransform>* slice_transform) { + // While we normally don't convert the string representation of a + // pointer-typed option into its instance, here we do so for backward + // compatibility as we allow this action in SetOption(). + + // TODO(yhchiang): A possible better place for these serialization / + // deserialization is inside the class definition of pointer-typed + // option itself, but this requires a bigger change of public API. + bool result = + ParseSliceTransformHelper("fixed:", "capped:", value, slice_transform); + if (result) { + return result; + } + result = ParseSliceTransformHelper( + "rocksdb.FixedPrefix.", "rocksdb.CappedPrefix.", value, slice_transform); + if (result) { + return result; + } + // TODO(yhchiang): we can further support other default + // SliceTransforms here. + return false; +} + +bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, + const std::string& value) { + switch (opt_type) { + case OptionType::kBoolean: + *reinterpret_cast<bool*>(opt_address) = ParseBoolean("", value); + break; + case OptionType::kInt: + *reinterpret_cast<int*>(opt_address) = ParseInt(value); + break; + case OptionType::kInt32T: + *reinterpret_cast<int32_t*>(opt_address) = ParseInt32(value); + break; + case OptionType::kInt64T: + PutUnaligned(reinterpret_cast<int64_t*>(opt_address), ParseInt64(value)); + break; + case OptionType::kVectorInt: + *reinterpret_cast<std::vector<int>*>(opt_address) = ParseVectorInt(value); + break; + case OptionType::kUInt: + *reinterpret_cast<unsigned int*>(opt_address) = ParseUint32(value); + break; + case OptionType::kUInt32T: + *reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value); + break; + case OptionType::kUInt64T: + PutUnaligned(reinterpret_cast<uint64_t*>(opt_address), ParseUint64(value)); + break; + case OptionType::kSizeT: + PutUnaligned(reinterpret_cast<size_t*>(opt_address), ParseSizeT(value)); + break; + case OptionType::kString: + *reinterpret_cast<std::string*>(opt_address) = value; + break; + case OptionType::kDouble: + *reinterpret_cast<double*>(opt_address) = ParseDouble(value); + break; + case OptionType::kCompactionStyle: + return ParseEnum<CompactionStyle>( + compaction_style_string_map, value, + reinterpret_cast<CompactionStyle*>(opt_address)); + case OptionType::kCompactionPri: + return ParseEnum<CompactionPri>( + compaction_pri_string_map, value, + reinterpret_cast<CompactionPri*>(opt_address)); + case OptionType::kCompressionType: + return ParseEnum<CompressionType>( + compression_type_string_map, value, + reinterpret_cast<CompressionType*>(opt_address)); + case OptionType::kVectorCompressionType: + return ParseVectorCompressionType( + value, reinterpret_cast<std::vector<CompressionType>*>(opt_address)); + case OptionType::kSliceTransform: + return ParseSliceTransform( + value, reinterpret_cast<std::shared_ptr<const SliceTransform>*>( + opt_address)); + case OptionType::kChecksumType: + return ParseEnum<ChecksumType>( + checksum_type_string_map, value, + reinterpret_cast<ChecksumType*>(opt_address)); + case OptionType::kBlockBasedTableIndexType: + return ParseEnum<BlockBasedTableOptions::IndexType>( + block_base_table_index_type_string_map, value, + reinterpret_cast<BlockBasedTableOptions::IndexType*>(opt_address)); + case OptionType::kBlockBasedTableDataBlockIndexType: + return ParseEnum<BlockBasedTableOptions::DataBlockIndexType>( + block_base_table_data_block_index_type_string_map, value, + reinterpret_cast<BlockBasedTableOptions::DataBlockIndexType*>( + opt_address)); + case OptionType::kEncodingType: + return ParseEnum<EncodingType>( + encoding_type_string_map, value, + reinterpret_cast<EncodingType*>(opt_address)); + case OptionType::kWALRecoveryMode: + return ParseEnum<WALRecoveryMode>( + wal_recovery_mode_string_map, value, + reinterpret_cast<WALRecoveryMode*>(opt_address)); + case OptionType::kAccessHint: + return ParseEnum<DBOptions::AccessHint>( + access_hint_string_map, value, + reinterpret_cast<DBOptions::AccessHint*>(opt_address)); + case OptionType::kInfoLogLevel: + return ParseEnum<InfoLogLevel>( + info_log_level_string_map, value, + reinterpret_cast<InfoLogLevel*>(opt_address)); + case OptionType::kCompactionOptionsFIFO: { + if (!FIFOCompactionOptionsSpecialCase( + value, reinterpret_cast<CompactionOptionsFIFO*>(opt_address))) { + return ParseStructOptions<CompactionOptionsFIFO>( + value, reinterpret_cast<CompactionOptionsFIFO*>(opt_address), + fifo_compaction_options_type_info); + } + return true; + } + case OptionType::kLRUCacheOptions: { + return ParseStructOptions<LRUCacheOptions>(value, + reinterpret_cast<LRUCacheOptions*>(opt_address), + lru_cache_options_type_info); + } + case OptionType::kCompactionOptionsUniversal: + return ParseStructOptions<CompactionOptionsUniversal>( + value, reinterpret_cast<CompactionOptionsUniversal*>(opt_address), + universal_compaction_options_type_info); + case OptionType::kCompactionStopStyle: + return ParseEnum<CompactionStopStyle>( + compaction_stop_style_string_map, value, + reinterpret_cast<CompactionStopStyle*>(opt_address)); + default: + return false; + } + return true; +} + +bool SerializeSingleOptionHelper(const char* opt_address, + const OptionType opt_type, + std::string* value) { + + assert(value); + switch (opt_type) { + case OptionType::kBoolean: + *value = *(reinterpret_cast<const bool*>(opt_address)) ? "true" : "false"; + break; + case OptionType::kInt: + *value = ToString(*(reinterpret_cast<const int*>(opt_address))); + break; + case OptionType::kInt32T: + *value = ToString(*(reinterpret_cast<const int32_t*>(opt_address))); + break; + case OptionType::kInt64T: + { + int64_t v; + GetUnaligned(reinterpret_cast<const int64_t*>(opt_address), &v); + *value = ToString(v); + } + break; + case OptionType::kVectorInt: + return SerializeIntVector( + *reinterpret_cast<const std::vector<int>*>(opt_address), value); + case OptionType::kUInt: + *value = ToString(*(reinterpret_cast<const unsigned int*>(opt_address))); + break; + case OptionType::kUInt32T: + *value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address))); + break; + case OptionType::kUInt64T: + { + uint64_t v; + GetUnaligned(reinterpret_cast<const uint64_t*>(opt_address), &v); + *value = ToString(v); + } + break; + case OptionType::kSizeT: + { + size_t v; + GetUnaligned(reinterpret_cast<const size_t*>(opt_address), &v); + *value = ToString(v); + } + break; + case OptionType::kDouble: + *value = ToString(*(reinterpret_cast<const double*>(opt_address))); + break; + case OptionType::kString: + *value = EscapeOptionString( + *(reinterpret_cast<const std::string*>(opt_address))); + break; + case OptionType::kCompactionStyle: + return SerializeEnum<CompactionStyle>( + compaction_style_string_map, + *(reinterpret_cast<const CompactionStyle*>(opt_address)), value); + case OptionType::kCompactionPri: + return SerializeEnum<CompactionPri>( + compaction_pri_string_map, + *(reinterpret_cast<const CompactionPri*>(opt_address)), value); + case OptionType::kCompressionType: + return SerializeEnum<CompressionType>( + compression_type_string_map, + *(reinterpret_cast<const CompressionType*>(opt_address)), value); + case OptionType::kVectorCompressionType: + return SerializeVectorCompressionType( + *(reinterpret_cast<const std::vector<CompressionType>*>(opt_address)), + value); + break; + case OptionType::kSliceTransform: { + const auto* slice_transform_ptr = + reinterpret_cast<const std::shared_ptr<const SliceTransform>*>( + opt_address); + *value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name() + : kNullptrString; + break; + } + case OptionType::kTableFactory: { + const auto* table_factory_ptr = + reinterpret_cast<const std::shared_ptr<const TableFactory>*>( + opt_address); + *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name() + : kNullptrString; + break; + } + case OptionType::kComparator: { + // it's a const pointer of const Comparator* + const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address); + // Since the user-specified comparator will be wrapped by + // InternalKeyComparator, we should persist the user-specified one + // instead of InternalKeyComparator. + if (*ptr == nullptr) { + *value = kNullptrString; + } else { + const Comparator* root_comp = (*ptr)->GetRootComparator(); + if (root_comp == nullptr) { + root_comp = (*ptr); + } + *value = root_comp->Name(); + } + break; + } + case OptionType::kCompactionFilter: { + // it's a const pointer of const CompactionFilter* + const auto* ptr = + reinterpret_cast<const CompactionFilter* const*>(opt_address); + *value = *ptr ? (*ptr)->Name() : kNullptrString; + break; + } + case OptionType::kCompactionFilterFactory: { + const auto* ptr = + reinterpret_cast<const std::shared_ptr<CompactionFilterFactory>*>( + opt_address); + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; + break; + } + case OptionType::kMemTableRepFactory: { + const auto* ptr = + reinterpret_cast<const std::shared_ptr<MemTableRepFactory>*>( + opt_address); + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; + break; + } + case OptionType::kMergeOperator: { + const auto* ptr = + reinterpret_cast<const std::shared_ptr<MergeOperator>*>(opt_address); + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; + break; + } + case OptionType::kFilterPolicy: { + const auto* ptr = + reinterpret_cast<const std::shared_ptr<FilterPolicy>*>(opt_address); + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; + break; + } + case OptionType::kChecksumType: + return SerializeEnum<ChecksumType>( + checksum_type_string_map, + *reinterpret_cast<const ChecksumType*>(opt_address), value); + case OptionType::kBlockBasedTableIndexType: + return SerializeEnum<BlockBasedTableOptions::IndexType>( + block_base_table_index_type_string_map, + *reinterpret_cast<const BlockBasedTableOptions::IndexType*>( + opt_address), + value); + case OptionType::kBlockBasedTableDataBlockIndexType: + return SerializeEnum<BlockBasedTableOptions::DataBlockIndexType>( + block_base_table_data_block_index_type_string_map, + *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>( + opt_address), + value); + case OptionType::kFlushBlockPolicyFactory: { + const auto* ptr = + reinterpret_cast<const std::shared_ptr<FlushBlockPolicyFactory>*>( + opt_address); + *value = ptr->get() ? ptr->get()->Name() : kNullptrString; + break; + } + case OptionType::kEncodingType: + return SerializeEnum<EncodingType>( + encoding_type_string_map, + *reinterpret_cast<const EncodingType*>(opt_address), value); + case OptionType::kWALRecoveryMode: + return SerializeEnum<WALRecoveryMode>( + wal_recovery_mode_string_map, + *reinterpret_cast<const WALRecoveryMode*>(opt_address), value); + case OptionType::kAccessHint: + return SerializeEnum<DBOptions::AccessHint>( + access_hint_string_map, + *reinterpret_cast<const DBOptions::AccessHint*>(opt_address), value); + case OptionType::kInfoLogLevel: + return SerializeEnum<InfoLogLevel>( + info_log_level_string_map, + *reinterpret_cast<const InfoLogLevel*>(opt_address), value); + case OptionType::kCompactionOptionsFIFO: + return SerializeStruct<CompactionOptionsFIFO>( + *reinterpret_cast<const CompactionOptionsFIFO*>(opt_address), value, + fifo_compaction_options_type_info); + case OptionType::kCompactionOptionsUniversal: + return SerializeStruct<CompactionOptionsUniversal>( + *reinterpret_cast<const CompactionOptionsUniversal*>(opt_address), + value, universal_compaction_options_type_info); + case OptionType::kCompactionStopStyle: + return SerializeEnum<CompactionStopStyle>( + compaction_stop_style_string_map, + *reinterpret_cast<const CompactionStopStyle*>(opt_address), value); + default: + return false; + } + return true; +} + +Status GetMutableOptionsFromStrings( + const MutableCFOptions& base_options, + const std::unordered_map<std::string, std::string>& options_map, + Logger* info_log, MutableCFOptions* new_options) { + assert(new_options); + *new_options = base_options; + for (const auto& o : options_map) { + try { + auto iter = cf_options_type_info.find(o.first); + if (iter == cf_options_type_info.end()) { + return Status::InvalidArgument("Unrecognized option: " + o.first); + } + const auto& opt_info = iter->second; + if (!opt_info.is_mutable) { + return Status::InvalidArgument("Option not changeable: " + o.first); + } + if (opt_info.verification == OptionVerificationType::kDeprecated) { + // log warning when user tries to set a deprecated option but don't fail + // the call for compatibility. + ROCKS_LOG_WARN(info_log, "%s is a deprecated option and cannot be set", + o.first.c_str()); + continue; + } + bool is_ok = ParseOptionHelper( + reinterpret_cast<char*>(new_options) + opt_info.mutable_offset, + opt_info.type, o.second); + if (!is_ok) { + return Status::InvalidArgument("Error parsing " + o.first); + } + } catch (std::exception& e) { + return Status::InvalidArgument("Error parsing " + o.first + ":" + + std::string(e.what())); + } + } + return Status::OK(); +} + +Status GetMutableDBOptionsFromStrings( + const MutableDBOptions& base_options, + const std::unordered_map<std::string, std::string>& options_map, + MutableDBOptions* new_options) { + assert(new_options); + *new_options = base_options; + for (const auto& o : options_map) { + try { + auto iter = db_options_type_info.find(o.first); + if (iter == db_options_type_info.end()) { + return Status::InvalidArgument("Unrecognized option: " + o.first); + } + const auto& opt_info = iter->second; + if (!opt_info.is_mutable) { + return Status::InvalidArgument("Option not changeable: " + o.first); + } + bool is_ok = ParseOptionHelper( + reinterpret_cast<char*>(new_options) + opt_info.mutable_offset, + opt_info.type, o.second); + if (!is_ok) { + return Status::InvalidArgument("Error parsing " + o.first); + } + } catch (std::exception& e) { + return Status::InvalidArgument("Error parsing " + o.first + ":" + + std::string(e.what())); + } + } + return Status::OK(); +} + +Status StringToMap(const std::string& opts_str, + std::unordered_map<std::string, std::string>* opts_map) { + assert(opts_map); + // Example: + // opts_str = "write_buffer_size=1024;max_write_buffer_number=2;" + // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100" + size_t pos = 0; + std::string opts = trim(opts_str); + while (pos < opts.size()) { + size_t eq_pos = opts.find('=', pos); + if (eq_pos == std::string::npos) { + return Status::InvalidArgument("Mismatched key value pair, '=' expected"); + } + std::string key = trim(opts.substr(pos, eq_pos - pos)); + if (key.empty()) { + return Status::InvalidArgument("Empty key found"); + } + + // skip space after '=' and look for '{' for possible nested options + pos = eq_pos + 1; + while (pos < opts.size() && isspace(opts[pos])) { + ++pos; + } + // Empty value at the end + if (pos >= opts.size()) { + (*opts_map)[key] = ""; + break; + } + if (opts[pos] == '{') { + int count = 1; + size_t brace_pos = pos + 1; + while (brace_pos < opts.size()) { + if (opts[brace_pos] == '{') { + ++count; + } else if (opts[brace_pos] == '}') { + --count; + if (count == 0) { + break; + } + } + ++brace_pos; + } + // found the matching closing brace + if (count == 0) { + (*opts_map)[key] = trim(opts.substr(pos + 1, brace_pos - pos - 1)); + // skip all whitespace and move to the next ';' + // brace_pos points to the next position after the matching '}' + pos = brace_pos + 1; + while (pos < opts.size() && isspace(opts[pos])) { + ++pos; + } + if (pos < opts.size() && opts[pos] != ';') { + return Status::InvalidArgument( + "Unexpected chars after nested options"); + } + ++pos; + } else { + return Status::InvalidArgument( + "Mismatched curly braces for nested options"); + } + } else { + size_t sc_pos = opts.find(';', pos); + if (sc_pos == std::string::npos) { + (*opts_map)[key] = trim(opts.substr(pos)); + // It either ends with a trailing semi-colon or the last key-value pair + break; + } else { + (*opts_map)[key] = trim(opts.substr(pos, sc_pos - pos)); + } + pos = sc_pos + 1; + } + } + + return Status::OK(); +} + +Status ParseCompressionOptions(const std::string& value, const std::string& name, + CompressionOptions& compression_opts) { + size_t start = 0; + size_t end = value.find(':'); + if (end == std::string::npos) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + compression_opts.window_bits = ParseInt(value.substr(start, end - start)); + start = end + 1; + end = value.find(':', start); + if (end == std::string::npos) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + compression_opts.level = ParseInt(value.substr(start, end - start)); + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument("unable to parse the specified CF option " + + name); + } + end = value.find(':', start); + compression_opts.strategy = + ParseInt(value.substr(start, value.size() - start)); + // max_dict_bytes is optional for backwards compatibility + if (end != std::string::npos) { + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.max_dict_bytes = + ParseInt(value.substr(start, value.size() - start)); + end = value.find(':', start); + } + // zstd_max_train_bytes is optional for backwards compatibility + if (end != std::string::npos) { + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.zstd_max_train_bytes = + ParseInt(value.substr(start, value.size() - start)); + end = value.find(':', start); + } + // enabled is optional for backwards compatibility + if (end != std::string::npos) { + start = end + 1; + if (start >= value.size()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + compression_opts.enabled = + ParseBoolean("", value.substr(start, value.size() - start)); + } + return Status::OK(); +} + +Status ParseColumnFamilyOption(const std::string& name, + const std::string& org_value, + ColumnFamilyOptions* new_options, + bool input_strings_escaped = false) { + const std::string& value = + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + try { + if (name == "block_based_table_factory") { + // Nested options + BlockBasedTableOptions table_opt, base_table_options; + BlockBasedTableFactory* block_based_table_factory = + static_cast_with_check<BlockBasedTableFactory, TableFactory>( + new_options->table_factory.get()); + if (block_based_table_factory != nullptr) { + base_table_options = block_based_table_factory->table_options(); + } + Status table_opt_s = GetBlockBasedTableOptionsFromString( + base_table_options, value, &table_opt); + if (!table_opt_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); + } else if (name == "plain_table_factory") { + // Nested options + PlainTableOptions table_opt, base_table_options; + PlainTableFactory* plain_table_factory = + static_cast_with_check<PlainTableFactory, TableFactory>( + new_options->table_factory.get()); + if (plain_table_factory != nullptr) { + base_table_options = plain_table_factory->table_options(); + } + Status table_opt_s = GetPlainTableOptionsFromString( + base_table_options, value, &table_opt); + if (!table_opt_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->table_factory.reset(NewPlainTableFactory(table_opt)); + } else if (name == "memtable") { + std::unique_ptr<MemTableRepFactory> new_mem_factory; + Status mem_factory_s = + GetMemTableRepFactoryFromString(value, &new_mem_factory); + if (!mem_factory_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->memtable_factory.reset(new_mem_factory.release()); + } else if (name == "bottommost_compression_opts") { + Status s = ParseCompressionOptions( + value, name, new_options->bottommost_compression_opts); + if (!s.ok()) { + return s; + } + } else if (name == "compression_opts") { + Status s = + ParseCompressionOptions(value, name, new_options->compression_opts); + if (!s.ok()) { + return s; + } + } else { + if (name == kNameComparator) { + // Try to get comparator from object registry first. + std::unique_ptr<const Comparator> comp_guard; + const Comparator* comp = + NewCustomObject<const Comparator>(value, &comp_guard); + // Only support static comparator for now. + if (comp != nullptr && !comp_guard) { + new_options->comparator = comp; + } + } else if (name == kNameMergeOperator) { + // Try to get merge operator from object registry first. + std::unique_ptr<std::shared_ptr<MergeOperator>> mo_guard; + std::shared_ptr<MergeOperator>* mo = + NewCustomObject<std::shared_ptr<MergeOperator>>(value, &mo_guard); + // Only support static comparator for now. + if (mo != nullptr) { + new_options->merge_operator = *mo; + } + } + + auto iter = cf_options_type_info.find(name); + if (iter == cf_options_type_info.end()) { + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); + } + const auto& opt_info = iter->second; + if (opt_info.verification != OptionVerificationType::kDeprecated && + ParseOptionHelper( + reinterpret_cast<char*>(new_options) + opt_info.offset, + opt_info.type, value)) { + return Status::OK(); + } + switch (opt_info.verification) { + case OptionVerificationType::kByName: + case OptionVerificationType::kByNameAllowNull: + case OptionVerificationType::kByNameAllowFromNull: + return Status::NotSupported( + "Deserializing the specified CF option " + name + + " is not supported"); + case OptionVerificationType::kDeprecated: + return Status::OK(); + default: + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); + } + } + } catch (const std::exception&) { + return Status::InvalidArgument( + "unable to parse the specified option " + name); + } + return Status::OK(); +} + +template <typename T> +bool SerializeSingleStructOption( + std::string* opt_string, const T& options, + const std::unordered_map<std::string, OptionTypeInfo> type_info, + const std::string& name, const std::string& delimiter) { + auto iter = type_info.find(name); + if (iter == type_info.end()) { + return false; + } + auto& opt_info = iter->second; + const char* opt_address = + reinterpret_cast<const char*>(&options) + opt_info.offset; + std::string value; + bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); + if (result) { + *opt_string = name + "=" + value + delimiter; + } + return result; +} + +template <typename T> +Status GetStringFromStruct( + std::string* opt_string, const T& options, + const std::unordered_map<std::string, OptionTypeInfo> type_info, + const std::string& delimiter) { + assert(opt_string); + opt_string->clear(); + for (auto iter = type_info.begin(); iter != type_info.end(); ++iter) { + if (iter->second.verification == OptionVerificationType::kDeprecated) { + // If the option is no longer used in rocksdb and marked as deprecated, + // we skip it in the serialization. + continue; + } + std::string single_output; + bool result = SerializeSingleStructOption<T>( + &single_output, options, type_info, iter->first, delimiter); + if (result) { + opt_string->append(single_output); + } else { + return Status::InvalidArgument("failed to serialize %s\n", + iter->first.c_str()); + } + assert(result); + } + return Status::OK(); +} + +Status GetStringFromDBOptions(std::string* opt_string, + const DBOptions& db_options, + const std::string& delimiter) { + return GetStringFromStruct<DBOptions>(opt_string, db_options, + db_options_type_info, delimiter); +} + +Status GetStringFromColumnFamilyOptions(std::string* opt_string, + const ColumnFamilyOptions& cf_options, + const std::string& delimiter) { + return GetStringFromStruct<ColumnFamilyOptions>( + opt_string, cf_options, cf_options_type_info, delimiter); +} + +Status GetStringFromCompressionType(std::string* compression_str, + CompressionType compression_type) { + bool ok = SerializeEnum<CompressionType>(compression_type_string_map, + compression_type, compression_str); + if (ok) { + return Status::OK(); + } else { + return Status::InvalidArgument("Invalid compression types"); + } +} + +std::vector<CompressionType> GetSupportedCompressions() { + std::vector<CompressionType> supported_compressions; + for (const auto& comp_to_name : compression_type_string_map) { + CompressionType t = comp_to_name.second; + if (t != kDisableCompressionOption && CompressionTypeSupported(t)) { + supported_compressions.push_back(t); + } + } + return supported_compressions; +} + +Status ParseDBOption(const std::string& name, + const std::string& org_value, + DBOptions* new_options, + bool input_strings_escaped = false) { + const std::string& value = + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + try { + if (name == "rate_limiter_bytes_per_sec") { + new_options->rate_limiter.reset( + NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value)))); + } else { + auto iter = db_options_type_info.find(name); + if (iter == db_options_type_info.end()) { + return Status::InvalidArgument("Unrecognized option DBOptions:", name); + } + const auto& opt_info = iter->second; + if (opt_info.verification != OptionVerificationType::kDeprecated && + ParseOptionHelper( + reinterpret_cast<char*>(new_options) + opt_info.offset, + opt_info.type, value)) { + return Status::OK(); + } + switch (opt_info.verification) { + case OptionVerificationType::kByName: + case OptionVerificationType::kByNameAllowNull: + return Status::NotSupported( + "Deserializing the specified DB option " + name + + " is not supported"); + case OptionVerificationType::kDeprecated: + return Status::OK(); + default: + return Status::InvalidArgument( + "Unable to parse the specified DB option " + name); + } + } + } catch (const std::exception&) { + return Status::InvalidArgument("Unable to parse DBOptions:", name); + } + return Status::OK(); +} + +Status GetColumnFamilyOptionsFromMap( + const ColumnFamilyOptions& base_options, + const std::unordered_map<std::string, std::string>& opts_map, + ColumnFamilyOptions* new_options, bool input_strings_escaped, + bool ignore_unknown_options) { + return GetColumnFamilyOptionsFromMapInternal( + base_options, opts_map, new_options, input_strings_escaped, nullptr, + ignore_unknown_options); +} + +Status GetColumnFamilyOptionsFromMapInternal( + const ColumnFamilyOptions& base_options, + const std::unordered_map<std::string, std::string>& opts_map, + ColumnFamilyOptions* new_options, bool input_strings_escaped, + std::vector<std::string>* unsupported_options_names, + bool ignore_unknown_options) { + assert(new_options); + *new_options = base_options; + if (unsupported_options_names) { + unsupported_options_names->clear(); + } + for (const auto& o : opts_map) { + auto s = ParseColumnFamilyOption(o.first, o.second, new_options, + input_strings_escaped); + if (!s.ok()) { + if (s.IsNotSupported()) { + // If the deserialization of the specified option is not supported + // and an output vector of unsupported_options is provided, then + // we log the name of the unsupported option and proceed. + if (unsupported_options_names != nullptr) { + unsupported_options_names->push_back(o.first); + } + // Note that we still return Status::OK in such case to maintain + // the backward compatibility in the old public API defined in + // rocksdb/convenience.h + } else if (s.IsInvalidArgument() && ignore_unknown_options) { + continue; + } else { + // Restore "new_options" to the default "base_options". + *new_options = base_options; + return s; + } + } + } + return Status::OK(); +} + +Status GetColumnFamilyOptionsFromString( + const ColumnFamilyOptions& base_options, + const std::string& opts_str, + ColumnFamilyOptions* new_options) { + std::unordered_map<std::string, std::string> opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + *new_options = base_options; + return s; + } + return GetColumnFamilyOptionsFromMap(base_options, opts_map, new_options); +} + +Status GetDBOptionsFromMap( + const DBOptions& base_options, + const std::unordered_map<std::string, std::string>& opts_map, + DBOptions* new_options, bool input_strings_escaped, + bool ignore_unknown_options) { + return GetDBOptionsFromMapInternal(base_options, opts_map, new_options, + input_strings_escaped, nullptr, + ignore_unknown_options); +} + +Status GetDBOptionsFromMapInternal( + const DBOptions& base_options, + const std::unordered_map<std::string, std::string>& opts_map, + DBOptions* new_options, bool input_strings_escaped, + std::vector<std::string>* unsupported_options_names, + bool ignore_unknown_options) { + assert(new_options); + *new_options = base_options; + if (unsupported_options_names) { + unsupported_options_names->clear(); + } + for (const auto& o : opts_map) { + auto s = ParseDBOption(o.first, o.second, + new_options, input_strings_escaped); + if (!s.ok()) { + if (s.IsNotSupported()) { + // If the deserialization of the specified option is not supported + // and an output vector of unsupported_options is provided, then + // we log the name of the unsupported option and proceed. + if (unsupported_options_names != nullptr) { + unsupported_options_names->push_back(o.first); + } + // Note that we still return Status::OK in such case to maintain + // the backward compatibility in the old public API defined in + // rocksdb/convenience.h + } else if (s.IsInvalidArgument() && ignore_unknown_options) { + continue; + } else { + // Restore "new_options" to the default "base_options". + *new_options = base_options; + return s; + } + } + } + return Status::OK(); +} + +Status GetDBOptionsFromString( + const DBOptions& base_options, + const std::string& opts_str, + DBOptions* new_options) { + std::unordered_map<std::string, std::string> opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + *new_options = base_options; + return s; + } + return GetDBOptionsFromMap(base_options, opts_map, new_options); +} + +Status GetOptionsFromString(const Options& base_options, + const std::string& opts_str, Options* new_options) { + std::unordered_map<std::string, std::string> opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + return s; + } + DBOptions new_db_options(base_options); + ColumnFamilyOptions new_cf_options(base_options); + for (const auto& o : opts_map) { + if (ParseDBOption(o.first, o.second, &new_db_options).ok()) { + } else if (ParseColumnFamilyOption( + o.first, o.second, &new_cf_options).ok()) { + } else { + return Status::InvalidArgument("Can't parse option " + o.first); + } + } + *new_options = Options(new_db_options, new_cf_options); + return Status::OK(); +} + +Status GetTableFactoryFromMap( + const std::string& factory_name, + const std::unordered_map<std::string, std::string>& opt_map, + std::shared_ptr<TableFactory>* table_factory, bool ignore_unknown_options) { + Status s; + if (factory_name == BlockBasedTableFactory().Name()) { + BlockBasedTableOptions bbt_opt; + s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map, + &bbt_opt, + true, /* input_strings_escaped */ + ignore_unknown_options); + if (!s.ok()) { + return s; + } + table_factory->reset(new BlockBasedTableFactory(bbt_opt)); + return Status::OK(); + } else if (factory_name == PlainTableFactory().Name()) { + PlainTableOptions pt_opt; + s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt, + true, /* input_strings_escaped */ + ignore_unknown_options); + if (!s.ok()) { + return s; + } + table_factory->reset(new PlainTableFactory(pt_opt)); + return Status::OK(); + } + // Return OK for not supported table factories as TableFactory + // Deserialization is optional. + table_factory->reset(); + return Status::OK(); +} + +std::unordered_map<std::string, OptionTypeInfo> + OptionsHelper::db_options_type_info = { + /* + // not yet supported + Env* env; + std::shared_ptr<Cache> row_cache; + std::shared_ptr<DeleteScheduler> delete_scheduler; + std::shared_ptr<Logger> info_log; + std::shared_ptr<RateLimiter> rate_limiter; + std::shared_ptr<Statistics> statistics; + std::vector<DbPath> db_paths; + std::vector<std::shared_ptr<EventListener>> listeners; + */ + {"advise_random_on_open", + {offsetof(struct DBOptions, advise_random_on_open), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"allow_mmap_reads", + {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"allow_fallocate", + {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"allow_mmap_writes", + {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"use_direct_reads", + {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"use_direct_writes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, + 0}}, + {"use_direct_io_for_flush_and_compaction", + {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"allow_2pc", + {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"allow_os_buffer", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, + 0}}, + {"create_if_missing", + {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"create_missing_column_families", + {offsetof(struct DBOptions, create_missing_column_families), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"disableDataSync", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, + 0}}, + {"disable_data_sync", // for compatibility + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, + 0}}, + {"enable_thread_tracking", + {offsetof(struct DBOptions, enable_thread_tracking), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"error_if_exists", + {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"is_fd_close_on_exec", + {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"paranoid_checks", + {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"skip_log_error_on_recovery", + {offsetof(struct DBOptions, skip_log_error_on_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"skip_stats_update_on_db_open", + {offsetof(struct DBOptions, skip_stats_update_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"new_table_reader_for_compaction_inputs", + {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"compaction_readahead_size", + {offsetof(struct DBOptions, compaction_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, compaction_readahead_size)}}, + {"random_access_max_buffer_size", + {offsetof(struct DBOptions, random_access_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, + {"use_adaptive_mutex", + {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"use_fsync", + {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"max_background_jobs", + {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, max_background_jobs)}}, + {"max_background_compactions", + {offsetof(struct DBOptions, max_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, max_background_compactions)}}, + {"base_background_compactions", + {offsetof(struct DBOptions, base_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, base_background_compactions)}}, + {"max_background_flushes", + {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, + OptionVerificationType::kNormal, false, 0}}, + {"max_file_opening_threads", + {offsetof(struct DBOptions, max_file_opening_threads), + OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, + {"max_open_files", + {offsetof(struct DBOptions, max_open_files), OptionType::kInt, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, max_open_files)}}, + {"table_cache_numshardbits", + {offsetof(struct DBOptions, table_cache_numshardbits), + OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, + {"db_write_buffer_size", + {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, + OptionVerificationType::kNormal, false, 0}}, + {"keep_log_file_num", + {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal, false, 0}}, + {"recycle_log_file_num", + {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal, false, 0}}, + {"log_file_time_to_roll", + {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, + OptionVerificationType::kNormal, false, 0}}, + {"manifest_preallocation_size", + {offsetof(struct DBOptions, manifest_preallocation_size), + OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, + {"max_log_file_size", + {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, + OptionVerificationType::kNormal, false, 0}}, + {"db_log_dir", + {offsetof(struct DBOptions, db_log_dir), OptionType::kString, + OptionVerificationType::kNormal, false, 0}}, + {"wal_dir", + {offsetof(struct DBOptions, wal_dir), OptionType::kString, + OptionVerificationType::kNormal, false, 0}}, + {"max_subcompactions", + {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T, + OptionVerificationType::kNormal, false, 0}}, + {"WAL_size_limit_MB", + {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, + OptionVerificationType::kNormal, false, 0}}, + {"WAL_ttl_seconds", + {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, + OptionVerificationType::kNormal, false, 0}}, + {"bytes_per_sync", + {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, bytes_per_sync)}}, + {"delayed_write_rate", + {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, delayed_write_rate)}}, + {"delete_obsolete_files_period_micros", + {offsetof(struct DBOptions, delete_obsolete_files_period_micros), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, + delete_obsolete_files_period_micros)}}, + {"max_manifest_file_size", + {offsetof(struct DBOptions, max_manifest_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, + {"max_total_wal_size", + {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, max_total_wal_size)}}, + {"wal_bytes_per_sync", + {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, wal_bytes_per_sync)}}, + {"stats_dump_period_sec", + {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt, + OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, stats_dump_period_sec)}}, + {"stats_persist_period_sec", + {offsetof(struct DBOptions, stats_persist_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, stats_persist_period_sec)}}, + {"stats_history_buffer_size", + {offsetof(struct DBOptions, stats_history_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, stats_history_buffer_size)}}, + {"fail_if_options_file_error", + {offsetof(struct DBOptions, fail_if_options_file_error), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"enable_pipelined_write", + {offsetof(struct DBOptions, enable_pipelined_write), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"allow_concurrent_memtable_write", + {offsetof(struct DBOptions, allow_concurrent_memtable_write), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"wal_recovery_mode", + {offsetof(struct DBOptions, wal_recovery_mode), + OptionType::kWALRecoveryMode, OptionVerificationType::kNormal, false, + 0}}, + {"enable_write_thread_adaptive_yield", + {offsetof(struct DBOptions, enable_write_thread_adaptive_yield), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"write_thread_slow_yield_usec", + {offsetof(struct DBOptions, write_thread_slow_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, + {"write_thread_max_yield_usec", + {offsetof(struct DBOptions, write_thread_max_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, + {"access_hint_on_compaction_start", + {offsetof(struct DBOptions, access_hint_on_compaction_start), + OptionType::kAccessHint, OptionVerificationType::kNormal, false, 0}}, + {"info_log_level", + {offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel, + OptionVerificationType::kNormal, false, 0}}, + {"dump_malloc_stats", + {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean, + OptionVerificationType::kNormal, false, 0}}, + {"avoid_flush_during_recovery", + {offsetof(struct DBOptions, avoid_flush_during_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"avoid_flush_during_shutdown", + {offsetof(struct DBOptions, avoid_flush_during_shutdown), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}, + {"writable_file_max_buffer_size", + {offsetof(struct DBOptions, writable_file_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, true, + offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}}, + {"allow_ingest_behind", + {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean, + OptionVerificationType::kNormal, false, + offsetof(struct ImmutableDBOptions, allow_ingest_behind)}}, + {"preserve_deletes", + {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean, + OptionVerificationType::kNormal, false, + offsetof(struct ImmutableDBOptions, preserve_deletes)}}, + {"concurrent_prepare", // Deprecated by two_write_queues + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, + 0}}, + {"two_write_queues", + {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean, + OptionVerificationType::kNormal, false, + offsetof(struct ImmutableDBOptions, two_write_queues)}}, + {"manual_wal_flush", + {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean, + OptionVerificationType::kNormal, false, + offsetof(struct ImmutableDBOptions, manual_wal_flush)}}, + {"seq_per_batch", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, + 0}}, + {"atomic_flush", + {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean, + OptionVerificationType::kNormal, false, + offsetof(struct ImmutableDBOptions, atomic_flush)}}, + {"avoid_unnecessary_blocking_io", + {offsetof(struct DBOptions, avoid_unnecessary_blocking_io), + OptionType::kBoolean, OptionVerificationType::kNormal, false, + offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}} + }; + +std::unordered_map<std::string, BlockBasedTableOptions::IndexType> + OptionsHelper::block_base_table_index_type_string_map = { + {"kBinarySearch", BlockBasedTableOptions::IndexType::kBinarySearch}, + {"kHashSearch", BlockBasedTableOptions::IndexType::kHashSearch}, + {"kTwoLevelIndexSearch", + BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch}}; + +std::unordered_map<std::string, BlockBasedTableOptions::DataBlockIndexType> + OptionsHelper::block_base_table_data_block_index_type_string_map = { + {"kDataBlockBinarySearch", + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinarySearch}, + {"kDataBlockBinaryAndHash", + BlockBasedTableOptions::DataBlockIndexType::kDataBlockBinaryAndHash}}; + +std::unordered_map<std::string, EncodingType> + OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain}, + {"kPrefix", kPrefix}}; + +std::unordered_map<std::string, CompactionStyle> + OptionsHelper::compaction_style_string_map = { + {"kCompactionStyleLevel", kCompactionStyleLevel}, + {"kCompactionStyleUniversal", kCompactionStyleUniversal}, + {"kCompactionStyleFIFO", kCompactionStyleFIFO}, + {"kCompactionStyleNone", kCompactionStyleNone}}; + +std::unordered_map<std::string, CompactionPri> + OptionsHelper::compaction_pri_string_map = { + {"kByCompensatedSize", kByCompensatedSize}, + {"kOldestLargestSeqFirst", kOldestLargestSeqFirst}, + {"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst}, + {"kMinOverlappingRatio", kMinOverlappingRatio}}; + +std::unordered_map<std::string, WALRecoveryMode> + OptionsHelper::wal_recovery_mode_string_map = { + {"kTolerateCorruptedTailRecords", + WALRecoveryMode::kTolerateCorruptedTailRecords}, + {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency}, + {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery}, + {"kSkipAnyCorruptedRecords", + WALRecoveryMode::kSkipAnyCorruptedRecords}}; + +std::unordered_map<std::string, DBOptions::AccessHint> + OptionsHelper::access_hint_string_map = { + {"NONE", DBOptions::AccessHint::NONE}, + {"NORMAL", DBOptions::AccessHint::NORMAL}, + {"SEQUENTIAL", DBOptions::AccessHint::SEQUENTIAL}, + {"WILLNEED", DBOptions::AccessHint::WILLNEED}}; + +std::unordered_map<std::string, InfoLogLevel> + OptionsHelper::info_log_level_string_map = { + {"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL}, + {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL}, + {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL}, + {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL}, + {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, + {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; + +ColumnFamilyOptions OptionsHelper::dummy_cf_options; +CompactionOptionsFIFO OptionsHelper::dummy_comp_options; +LRUCacheOptions OptionsHelper::dummy_lru_cache_options; +CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal; + +// offset_of is used to get the offset of a class data member +// ex: offset_of(&ColumnFamilyOptions::num_levels) +// This call will return the offset of num_levels in ColumnFamilyOptions class +// +// This is the same as offsetof() but allow us to work with non standard-layout +// classes and structures +// refs: +// http://en.cppreference.com/w/cpp/concept/StandardLayoutType +// https://gist.github.com/graphitemaster/494f21190bb2c63c5516 +template <typename T1> +int offset_of(T1 ColumnFamilyOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - + size_t(&OptionsHelper::dummy_cf_options)); +} +template <typename T1> +int offset_of(T1 AdvancedColumnFamilyOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - + size_t(&OptionsHelper::dummy_cf_options)); +} +template <typename T1> +int offset_of(T1 CompactionOptionsFIFO::*member) { + return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) - + size_t(&OptionsHelper::dummy_comp_options)); +} +template <typename T1> +int offset_of(T1 LRUCacheOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - + size_t(&OptionsHelper::dummy_lru_cache_options)); +} +template <typename T1> +int offset_of(T1 CompactionOptionsUniversal::*member) { + return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) - + size_t(&OptionsHelper::dummy_comp_options_universal)); +} + +std::unordered_map<std::string, OptionTypeInfo> + OptionsHelper::cf_options_type_info = { + /* not yet supported + CompressionOptions compression_opts; + TablePropertiesCollectorFactories table_properties_collector_factories; + typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>> + TablePropertiesCollectorFactories; + UpdateStatus (*inplace_callback)(char* existing_value, + uint34_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + std::vector<DbPath> cf_paths; + */ + {"report_bg_io_stats", + {offset_of(&ColumnFamilyOptions::report_bg_io_stats), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, report_bg_io_stats)}}, + {"compaction_measure_io_stats", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, + 0}}, + {"disable_auto_compactions", + {offset_of(&ColumnFamilyOptions::disable_auto_compactions), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, disable_auto_compactions)}}, + {"filter_deletes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, + 0}}, + {"inplace_update_support", + {offset_of(&ColumnFamilyOptions::inplace_update_support), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"level_compaction_dynamic_level_bytes", + {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"optimize_filters_for_hits", + {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"paranoid_file_checks", + {offset_of(&ColumnFamilyOptions::paranoid_file_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, paranoid_file_checks)}}, + {"force_consistency_checks", + {offset_of(&ColumnFamilyOptions::force_consistency_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, + {"purge_redundant_kvs_while_flush", + {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), + OptionType::kBoolean, OptionVerificationType::kDeprecated, false, 0}}, + {"verify_checksums_in_compaction", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, + 0}}, + {"soft_pending_compaction_bytes_limit", + {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, + soft_pending_compaction_bytes_limit)}}, + {"hard_pending_compaction_bytes_limit", + {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, + hard_pending_compaction_bytes_limit)}}, + {"hard_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true, + 0}}, + {"soft_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true, + 0}}, + {"max_compaction_bytes", + {offset_of(&ColumnFamilyOptions::max_compaction_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, max_compaction_bytes)}}, + {"expanded_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, + {"level0_file_num_compaction_trigger", + {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, + level0_file_num_compaction_trigger)}}, + {"level0_slowdown_writes_trigger", + {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}}, + {"level0_stop_writes_trigger", + {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}}, + {"max_grandparent_overlap_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, + {"max_mem_compaction_level", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, false, 0}}, + {"max_write_buffer_number", + {offset_of(&ColumnFamilyOptions::max_write_buffer_number), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, max_write_buffer_number)}}, + {"max_write_buffer_number_to_maintain", + {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), + OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, + {"min_write_buffer_number_to_merge", + {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), + OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, + {"num_levels", + {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, + OptionVerificationType::kNormal, false, 0}}, + {"source_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, + {"target_file_size_multiplier", + {offset_of(&ColumnFamilyOptions::target_file_size_multiplier), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, target_file_size_multiplier)}}, + {"arena_block_size", + {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT, + OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, arena_block_size)}}, + {"inplace_update_num_locks", + {offset_of(&ColumnFamilyOptions::inplace_update_num_locks), + OptionType::kSizeT, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, inplace_update_num_locks)}}, + {"max_successive_merges", + {offset_of(&ColumnFamilyOptions::max_successive_merges), + OptionType::kSizeT, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, max_successive_merges)}}, + {"memtable_huge_page_size", + {offset_of(&ColumnFamilyOptions::memtable_huge_page_size), + OptionType::kSizeT, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, memtable_huge_page_size)}}, + {"memtable_prefix_bloom_huge_page_tlb_size", + {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, true, 0}}, + {"write_buffer_size", + {offset_of(&ColumnFamilyOptions::write_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, write_buffer_size)}}, + {"bloom_locality", + {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, + OptionVerificationType::kNormal, false, 0}}, + {"memtable_prefix_bloom_bits", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, + 0}}, + {"memtable_prefix_bloom_size_ratio", + {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}}, + {"memtable_prefix_bloom_probes", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, + 0}}, + {"memtable_whole_key_filtering", + {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}}, + {"min_partial_merge_operands", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, + 0}}, + {"max_bytes_for_level_base", + {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, max_bytes_for_level_base)}}, + {"max_bytes_for_level_multiplier", + {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier), + OptionType::kDouble, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}}, + {"max_bytes_for_level_multiplier_additional", + {offset_of( + &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional), + OptionType::kVectorInt, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, + max_bytes_for_level_multiplier_additional)}}, + {"max_sequential_skip_in_iterations", + {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, + max_sequential_skip_in_iterations)}}, + {"target_file_size_base", + {offset_of(&ColumnFamilyOptions::target_file_size_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, target_file_size_base)}}, + {"rate_limit_delay_max_milliseconds", + {0, OptionType::kUInt, OptionVerificationType::kDeprecated, false, 0}}, + {"compression", + {offset_of(&ColumnFamilyOptions::compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, compression)}}, + {"compression_per_level", + {offset_of(&ColumnFamilyOptions::compression_per_level), + OptionType::kVectorCompressionType, OptionVerificationType::kNormal, + false, 0}}, + {"bottommost_compression", + {offset_of(&ColumnFamilyOptions::bottommost_compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, false, + 0}}, + {kNameComparator, + {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, + OptionVerificationType::kByName, false, 0}}, + {"prefix_extractor", + {offset_of(&ColumnFamilyOptions::prefix_extractor), + OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, + true, offsetof(struct MutableCFOptions, prefix_extractor)}}, + {"memtable_insert_with_hint_prefix_extractor", + {offset_of( + &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), + OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, + false, 0}}, + {"memtable_factory", + {offset_of(&ColumnFamilyOptions::memtable_factory), + OptionType::kMemTableRepFactory, OptionVerificationType::kByName, + false, 0}}, + {"table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kTableFactory, OptionVerificationType::kByName, false, + 0}}, + {"compaction_filter", + {offset_of(&ColumnFamilyOptions::compaction_filter), + OptionType::kCompactionFilter, OptionVerificationType::kByName, false, + 0}}, + {"compaction_filter_factory", + {offset_of(&ColumnFamilyOptions::compaction_filter_factory), + OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, + false, 0}}, + {kNameMergeOperator, + {offset_of(&ColumnFamilyOptions::merge_operator), + OptionType::kMergeOperator, + OptionVerificationType::kByNameAllowFromNull, false, 0}}, + {"compaction_style", + {offset_of(&ColumnFamilyOptions::compaction_style), + OptionType::kCompactionStyle, OptionVerificationType::kNormal, false, + 0}}, + {"compaction_pri", + {offset_of(&ColumnFamilyOptions::compaction_pri), + OptionType::kCompactionPri, OptionVerificationType::kNormal, false, + 0}}, + {"compaction_options_fifo", + {offset_of(&ColumnFamilyOptions::compaction_options_fifo), + OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, + true, offsetof(struct MutableCFOptions, compaction_options_fifo)}}, + {"compaction_options_universal", + {offset_of(&ColumnFamilyOptions::compaction_options_universal), + OptionType::kCompactionOptionsUniversal, + OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, compaction_options_universal)}}, + {"ttl", + {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T, + OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, ttl)}}, + {"sample_for_compression", + {offset_of(&ColumnFamilyOptions::sample_for_compression), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, sample_for_compression)}}}; + +std::unordered_map<std::string, OptionTypeInfo> + OptionsHelper::fifo_compaction_options_type_info = { + {"max_table_files_size", + {offset_of(&CompactionOptionsFIFO::max_table_files_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, + {"ttl", + {0, OptionType::kUInt64T, + OptionVerificationType::kDeprecated, false, + 0}}, + {"allow_compaction", + {offset_of(&CompactionOptionsFIFO::allow_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; + +std::unordered_map<std::string, OptionTypeInfo> + OptionsHelper::universal_compaction_options_type_info = { + {"size_ratio", + {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt, + OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, size_ratio)}}, + {"min_merge_width", + {offset_of(&CompactionOptionsUniversal::min_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, min_merge_width)}}, + {"max_merge_width", + {offset_of(&CompactionOptionsUniversal::max_merge_width), + OptionType::kUInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, max_merge_width)}}, + {"max_size_amplification_percent", + {offset_of( + &CompactionOptionsUniversal::max_size_amplification_percent), + OptionType::kUInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, + max_size_amplification_percent)}}, + {"compression_size_percent", + {offset_of(&CompactionOptionsUniversal::compression_size_percent), + OptionType::kInt, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, + compression_size_percent)}}, + {"stop_style", + {offset_of(&CompactionOptionsUniversal::stop_style), + OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, + true, offsetof(class CompactionOptionsUniversal, stop_style)}}, + {"allow_trivial_move", + {offset_of(&CompactionOptionsUniversal::allow_trivial_move), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}}; + +std::unordered_map<std::string, CompactionStopStyle> + OptionsHelper::compaction_stop_style_string_map = { + {"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize}, + {"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}}; + +std::unordered_map<std::string, OptionTypeInfo> + OptionsHelper::lru_cache_options_type_info = { + {"capacity", + {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT, + OptionVerificationType::kNormal, true, + offsetof(struct LRUCacheOptions, capacity)}}, + {"num_shard_bits", + {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt, + OptionVerificationType::kNormal, true, + offsetof(struct LRUCacheOptions, num_shard_bits)}}, + {"strict_capacity_limit", + {offset_of(&LRUCacheOptions::strict_capacity_limit), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, + {"high_pri_pool_ratio", + {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble, + OptionVerificationType::kNormal, true, + offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; + +#endif // !ROCKSDB_LITE + +} // namespace rocksdb diff --git a/src/rocksdb/options/options_helper.h b/src/rocksdb/options/options_helper.h new file mode 100644 index 00000000..1d3d880a --- /dev/null +++ b/src/rocksdb/options/options_helper.h @@ -0,0 +1,226 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include <map> +#include <stdexcept> +#include <string> +#include <vector> + +#include "options/cf_options.h" +#include "options/db_options.h" +#include "rocksdb/options.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" +#include "rocksdb/universal_compaction.h" + +namespace rocksdb { + +DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, + const MutableDBOptions& mutable_db_options); + +ColumnFamilyOptions BuildColumnFamilyOptions( + const ColumnFamilyOptions& ioptions, + const MutableCFOptions& mutable_cf_options); + +#ifndef ROCKSDB_LITE + +Status GetMutableOptionsFromStrings( + const MutableCFOptions& base_options, + const std::unordered_map<std::string, std::string>& options_map, + Logger* info_log, MutableCFOptions* new_options); + +Status GetMutableDBOptionsFromStrings( + const MutableDBOptions& base_options, + const std::unordered_map<std::string, std::string>& options_map, + MutableDBOptions* new_options); + +Status GetTableFactoryFromMap( + const std::string& factory_name, + const std::unordered_map<std::string, std::string>& opt_map, + std::shared_ptr<TableFactory>* table_factory, + bool ignore_unknown_options = false); + +enum class OptionType { + kBoolean, + kInt, + kInt32T, + kInt64T, + kVectorInt, + kUInt, + kUInt32T, + kUInt64T, + kSizeT, + kString, + kDouble, + kCompactionStyle, + kCompactionPri, + kSliceTransform, + kCompressionType, + kVectorCompressionType, + kTableFactory, + kComparator, + kCompactionFilter, + kCompactionFilterFactory, + kCompactionOptionsFIFO, + kCompactionOptionsUniversal, + kCompactionStopStyle, + kMergeOperator, + kMemTableRepFactory, + kBlockBasedTableIndexType, + kBlockBasedTableDataBlockIndexType, + kFilterPolicy, + kFlushBlockPolicyFactory, + kChecksumType, + kEncodingType, + kWALRecoveryMode, + kAccessHint, + kInfoLogLevel, + kLRUCacheOptions, + kUnknown +}; + +enum class OptionVerificationType { + kNormal, + kByName, // The option is pointer typed so we can only verify + // based on it's name. + kByNameAllowNull, // Same as kByName, but it also allows the case + // where one of them is a nullptr. + kByNameAllowFromNull, // Same as kByName, but it also allows the case + // where the old option is nullptr. + kDeprecated // The option is no longer used in rocksdb. The RocksDB + // OptionsParser will still accept this option if it + // happen to exists in some Options file. However, + // the parser will not include it in serialization + // and verification processes. +}; + +// A struct for storing constant option information such as option name, +// option type, and offset. +struct OptionTypeInfo { + int offset; + OptionType type; + OptionVerificationType verification; + bool is_mutable; + int mutable_offset; +}; + +// A helper function that converts "opt_address" to a std::string +// based on the specified OptionType. +bool SerializeSingleOptionHelper(const char* opt_address, + const OptionType opt_type, std::string* value); + +// In addition to its public version defined in rocksdb/convenience.h, +// this further takes an optional output vector "unsupported_options_names", +// which stores the name of all the unsupported options specified in "opts_map". +Status GetDBOptionsFromMapInternal( + const DBOptions& base_options, + const std::unordered_map<std::string, std::string>& opts_map, + DBOptions* new_options, bool input_strings_escaped, + std::vector<std::string>* unsupported_options_names = nullptr, + bool ignore_unknown_options = false); + +// In addition to its public version defined in rocksdb/convenience.h, +// this further takes an optional output vector "unsupported_options_names", +// which stores the name of all the unsupported options specified in "opts_map". +Status GetColumnFamilyOptionsFromMapInternal( + const ColumnFamilyOptions& base_options, + const std::unordered_map<std::string, std::string>& opts_map, + ColumnFamilyOptions* new_options, bool input_strings_escaped, + std::vector<std::string>* unsupported_options_names = nullptr, + bool ignore_unknown_options = false); + +bool ParseSliceTransform( + const std::string& value, + std::shared_ptr<const SliceTransform>* slice_transform); + +extern Status StringToMap( + const std::string& opts_str, + std::unordered_map<std::string, std::string>* opts_map); + +extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, + const std::string& value); +#endif // !ROCKSDB_LITE + +struct OptionsHelper { + static std::map<CompactionStyle, std::string> compaction_style_to_string; + static std::map<CompactionPri, std::string> compaction_pri_to_string; + static std::map<CompactionStopStyle, std::string> + compaction_stop_style_to_string; + static std::unordered_map<std::string, ChecksumType> checksum_type_string_map; + static std::unordered_map<std::string, CompressionType> + compression_type_string_map; +#ifndef ROCKSDB_LITE + static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info; + static std::unordered_map<std::string, OptionTypeInfo> + fifo_compaction_options_type_info; + static std::unordered_map<std::string, OptionTypeInfo> + universal_compaction_options_type_info; + static std::unordered_map<std::string, CompactionStopStyle> + compaction_stop_style_string_map; + static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info; + static std::unordered_map<std::string, OptionTypeInfo> + lru_cache_options_type_info; + static std::unordered_map<std::string, BlockBasedTableOptions::IndexType> + block_base_table_index_type_string_map; + static std::unordered_map<std::string, + BlockBasedTableOptions::DataBlockIndexType> + block_base_table_data_block_index_type_string_map; + static std::unordered_map<std::string, EncodingType> encoding_type_string_map; + static std::unordered_map<std::string, CompactionStyle> + compaction_style_string_map; + static std::unordered_map<std::string, CompactionPri> + compaction_pri_string_map; + static std::unordered_map<std::string, WALRecoveryMode> + wal_recovery_mode_string_map; + static std::unordered_map<std::string, DBOptions::AccessHint> + access_hint_string_map; + static std::unordered_map<std::string, InfoLogLevel> + info_log_level_string_map; + static ColumnFamilyOptions dummy_cf_options; + static CompactionOptionsFIFO dummy_comp_options; + static LRUCacheOptions dummy_lru_cache_options; + static CompactionOptionsUniversal dummy_comp_options_universal; +#endif // !ROCKSDB_LITE +}; + +// Some aliasing +static auto& compaction_style_to_string = + OptionsHelper::compaction_style_to_string; +static auto& compaction_pri_to_string = OptionsHelper::compaction_pri_to_string; +static auto& compaction_stop_style_to_string = + OptionsHelper::compaction_stop_style_to_string; +static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map; +#ifndef ROCKSDB_LITE +static auto& cf_options_type_info = OptionsHelper::cf_options_type_info; +static auto& fifo_compaction_options_type_info = + OptionsHelper::fifo_compaction_options_type_info; +static auto& universal_compaction_options_type_info = + OptionsHelper::universal_compaction_options_type_info; +static auto& compaction_stop_style_string_map = + OptionsHelper::compaction_stop_style_string_map; +static auto& db_options_type_info = OptionsHelper::db_options_type_info; +static auto& lru_cache_options_type_info = + OptionsHelper::lru_cache_options_type_info; +static auto& compression_type_string_map = + OptionsHelper::compression_type_string_map; +static auto& block_base_table_index_type_string_map = + OptionsHelper::block_base_table_index_type_string_map; +static auto& block_base_table_data_block_index_type_string_map = + OptionsHelper::block_base_table_data_block_index_type_string_map; +static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map; +static auto& compaction_style_string_map = + OptionsHelper::compaction_style_string_map; +static auto& compaction_pri_string_map = + OptionsHelper::compaction_pri_string_map; +static auto& wal_recovery_mode_string_map = + OptionsHelper::wal_recovery_mode_string_map; +static auto& access_hint_string_map = OptionsHelper::access_hint_string_map; +static auto& info_log_level_string_map = + OptionsHelper::info_log_level_string_map; +#endif // !ROCKSDB_LITE + +} // namespace rocksdb diff --git a/src/rocksdb/options/options_parser.cc b/src/rocksdb/options/options_parser.cc new file mode 100644 index 00000000..2a85fa53 --- /dev/null +++ b/src/rocksdb/options/options_parser.cc @@ -0,0 +1,818 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifndef ROCKSDB_LITE + +#include "options/options_parser.h" + +#include <cmath> +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "options/options_helper.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "util/cast_util.h" +#include "util/file_reader_writer.h" +#include "util/string_util.h" +#include "util/sync_point.h" + +#include "port/port.h" + +namespace rocksdb { + +static const std::string option_file_header = + "# This is a RocksDB option file.\n" + "#\n" + "# For detailed file format spec, please refer to the example file\n" + "# in examples/rocksdb_option_file_example.ini\n" + "#\n" + "\n"; + +Status PersistRocksDBOptions(const DBOptions& db_opt, + const std::vector<std::string>& cf_names, + const std::vector<ColumnFamilyOptions>& cf_opts, + const std::string& file_name, Env* env) { + TEST_SYNC_POINT("PersistRocksDBOptions:start"); + if (cf_names.size() != cf_opts.size()) { + return Status::InvalidArgument( + "cf_names.size() and cf_opts.size() must be the same"); + } + std::unique_ptr<WritableFile> wf; + + Status s = env->NewWritableFile(file_name, &wf, EnvOptions()); + if (!s.ok()) { + return s; + } + std::unique_ptr<WritableFileWriter> writable; + writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(), + nullptr /* statistics */)); + + std::string options_file_content; + + writable->Append(option_file_header + "[" + + opt_section_titles[kOptionSectionVersion] + + "]\n" + " rocksdb_version=" + + ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) + + "." + ToString(ROCKSDB_PATCH) + "\n"); + writable->Append(" options_file_version=" + + ToString(ROCKSDB_OPTION_FILE_MAJOR) + "." + + ToString(ROCKSDB_OPTION_FILE_MINOR) + "\n"); + writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] + + "]\n "); + + s = GetStringFromDBOptions(&options_file_content, db_opt, "\n "); + if (!s.ok()) { + writable->Close(); + return s; + } + writable->Append(options_file_content + "\n"); + + for (size_t i = 0; i < cf_opts.size(); ++i) { + // CFOptions section + writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] + + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); + s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i], + "\n "); + if (!s.ok()) { + writable->Close(); + return s; + } + writable->Append(options_file_content + "\n"); + // TableOptions section + auto* tf = cf_opts[i].table_factory.get(); + if (tf != nullptr) { + writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] + + tf->Name() + " \"" + EscapeOptionString(cf_names[i]) + + "\"]\n "); + options_file_content.clear(); + s = tf->GetOptionString(&options_file_content, "\n "); + if (!s.ok()) { + return s; + } + writable->Append(options_file_content + "\n"); + } + } + writable->Sync(true /* use_fsync */); + writable->Close(); + + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + db_opt, cf_names, cf_opts, file_name, env); +} + +RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); } + +void RocksDBOptionsParser::Reset() { + db_opt_ = DBOptions(); + db_opt_map_.clear(); + cf_names_.clear(); + cf_opts_.clear(); + cf_opt_maps_.clear(); + has_version_section_ = false; + has_db_options_ = false; + has_default_cf_options_ = false; + for (int i = 0; i < 3; ++i) { + db_version[i] = 0; + opt_file_version[i] = 0; + } +} + +bool RocksDBOptionsParser::IsSection(const std::string& line) { + if (line.size() < 2) { + return false; + } + if (line[0] != '[' || line[line.size() - 1] != ']') { + return false; + } + return true; +} + +Status RocksDBOptionsParser::ParseSection(OptionSection* section, + std::string* title, + std::string* argument, + const std::string& line, + const int line_num) { + *section = kOptionSectionUnknown; + // A section is of the form [<SectionName> "<SectionArg>"], where + // "<SectionArg>" is optional. + size_t arg_start_pos = line.find("\""); + size_t arg_end_pos = line.rfind("\""); + // The following if-then check tries to identify whether the input + // section has the optional section argument. + if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) { + *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true); + *argument = UnescapeOptionString( + line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1)); + } else { + *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true); + *argument = ""; + } + for (int i = 0; i < kOptionSectionUnknown; ++i) { + if (title->find(opt_section_titles[i]) == 0) { + if (i == kOptionSectionVersion || i == kOptionSectionDBOptions || + i == kOptionSectionCFOptions) { + if (title->size() == opt_section_titles[i].size()) { + // if true, then it indicats equal + *section = static_cast<OptionSection>(i); + return CheckSection(*section, *argument, line_num); + } + } else if (i == kOptionSectionTableOptions) { + // This type of sections has a sufffix at the end of the + // section title + if (title->size() > opt_section_titles[i].size()) { + *section = static_cast<OptionSection>(i); + return CheckSection(*section, *argument, line_num); + } + } + } + } + return Status::InvalidArgument(std::string("Unknown section ") + line); +} + +Status RocksDBOptionsParser::InvalidArgument(const int line_num, + const std::string& message) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error] ", + message + " (at line " + ToString(line_num) + ")"); +} + +Status RocksDBOptionsParser::ParseStatement(std::string* name, + std::string* value, + const std::string& line, + const int line_num) { + size_t eq_pos = line.find("="); + if (eq_pos == std::string::npos) { + return InvalidArgument(line_num, "A valid statement must have a '='."); + } + + *name = TrimAndRemoveComment(line.substr(0, eq_pos), true); + *value = + TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1)); + if (name->empty()) { + return InvalidArgument(line_num, + "A valid statement must have a variable name."); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env, + bool ignore_unknown_options) { + Reset(); + + std::unique_ptr<SequentialFile> seq_file; + Status s = env->NewSequentialFile(file_name, &seq_file, EnvOptions()); + if (!s.ok()) { + return s; + } + + OptionSection section = kOptionSectionUnknown; + std::string title; + std::string argument; + std::unordered_map<std::string, std::string> opt_map; + std::istringstream iss; + std::string line; + bool has_data = true; + // we only support single-lined statement. + for (int line_num = 1; + ReadOneLine(&iss, seq_file.get(), &line, &has_data, &s); ++line_num) { + if (!s.ok()) { + return s; + } + line = TrimAndRemoveComment(line); + if (line.empty()) { + continue; + } + if (IsSection(line)) { + s = EndSection(section, title, argument, opt_map, ignore_unknown_options); + opt_map.clear(); + if (!s.ok()) { + return s; + } + + // If the option file is not generated by a higher minor version, + // there shouldn't be any unknown option. + if (ignore_unknown_options && section == kOptionSectionVersion) { + if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR && + db_version[1] <= ROCKSDB_MINOR)) { + ignore_unknown_options = false; + } + } + + s = ParseSection(§ion, &title, &argument, line, line_num); + if (!s.ok()) { + return s; + } + } else { + std::string name; + std::string value; + s = ParseStatement(&name, &value, line, line_num); + if (!s.ok()) { + return s; + } + opt_map.insert({name, value}); + } + } + + s = EndSection(section, title, argument, opt_map, ignore_unknown_options); + opt_map.clear(); + if (!s.ok()) { + return s; + } + return ValidityCheck(); +} + +Status RocksDBOptionsParser::CheckSection(const OptionSection section, + const std::string& section_arg, + const int line_num) { + if (section == kOptionSectionDBOptions) { + if (has_db_options_) { + return InvalidArgument( + line_num, + "More than one DBOption section found in the option config file"); + } + has_db_options_ = true; + } else if (section == kOptionSectionCFOptions) { + bool is_default_cf = (section_arg == kDefaultColumnFamilyName); + if (cf_opts_.size() == 0 && !is_default_cf) { + return InvalidArgument( + line_num, + "Default column family must be the first CFOptions section " + "in the option config file"); + } else if (cf_opts_.size() != 0 && is_default_cf) { + return InvalidArgument( + line_num, + "Default column family must be the first CFOptions section " + "in the optio/n config file"); + } else if (GetCFOptions(section_arg) != nullptr) { + return InvalidArgument( + line_num, + "Two identical column families found in option config file"); + } + has_default_cf_options_ |= is_default_cf; + } else if (section == kOptionSectionTableOptions) { + if (GetCFOptions(section_arg) == nullptr) { + return InvalidArgument( + line_num, std::string( + "Does not find a matched column family name in " + "TableOptions section. Column Family Name:") + + section_arg); + } + } else if (section == kOptionSectionVersion) { + if (has_version_section_) { + return InvalidArgument( + line_num, + "More than one Version section found in the option config file."); + } + has_version_section_ = true; + } + return Status::OK(); +} + +Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name, + const std::string& ver_string, + const int max_count, + int* version) { + int version_index = 0; + int current_number = 0; + int current_digit_count = 0; + bool has_dot = false; + for (int i = 0; i < max_count; ++i) { + version[i] = 0; + } + const int kBufferSize = 200; + char buffer[kBufferSize]; + for (size_t i = 0; i < ver_string.size(); ++i) { + if (ver_string[i] == '.') { + if (version_index >= max_count - 1) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s can only contains at most %d dots.", + ver_name.c_str(), max_count - 1); + return Status::InvalidArgument(buffer); + } + if (current_digit_count == 0) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s must have at least one digit before each dot.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + version[version_index++] = current_number; + current_number = 0; + current_digit_count = 0; + has_dot = true; + } else if (isdigit(ver_string[i])) { + current_number = current_number * 10 + (ver_string[i] - '0'); + current_digit_count++; + } else { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s can only contains dots and numbers.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + } + version[version_index] = current_number; + if (has_dot && current_digit_count == 0) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s must have at least one digit after each dot.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::EndSection( + const OptionSection section, const std::string& section_title, + const std::string& section_arg, + const std::unordered_map<std::string, std::string>& opt_map, + bool ignore_unknown_options) { + Status s; + if (section == kOptionSectionDBOptions) { + s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true, + ignore_unknown_options); + if (!s.ok()) { + return s; + } + db_opt_map_ = opt_map; + } else if (section == kOptionSectionCFOptions) { + // This condition should be ensured earlier in ParseSection + // so we make an assertion here. + assert(GetCFOptions(section_arg) == nullptr); + cf_names_.emplace_back(section_arg); + cf_opts_.emplace_back(); + s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map, + &cf_opts_.back(), true, + ignore_unknown_options); + if (!s.ok()) { + return s; + } + // keep the parsed string. + cf_opt_maps_.emplace_back(opt_map); + } else if (section == kOptionSectionTableOptions) { + assert(GetCFOptions(section_arg) != nullptr); + auto* cf_opt = GetCFOptionsImpl(section_arg); + if (cf_opt == nullptr) { + return Status::InvalidArgument( + "The specified column family must be defined before the " + "TableOptions section:", + section_arg); + } + // Ignore error as table factory deserialization is optional + s = GetTableFactoryFromMap( + section_title.substr( + opt_section_titles[kOptionSectionTableOptions].size()), + opt_map, &(cf_opt->table_factory), ignore_unknown_options); + if (!s.ok()) { + return s; + } + } else if (section == kOptionSectionVersion) { + for (const auto pair : opt_map) { + if (pair.first == "rocksdb_version") { + s = ParseVersionNumber(pair.first, pair.second, 3, db_version); + if (!s.ok()) { + return s; + } + } else if (pair.first == "options_file_version") { + s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version); + if (!s.ok()) { + return s; + } + if (opt_file_version[0] < 1) { + return Status::InvalidArgument( + "A valid options_file_version must be at least 1."); + } + } + } + } + return Status::OK(); +} + +Status RocksDBOptionsParser::ValidityCheck() { + if (!has_db_options_) { + return Status::Corruption( + "A RocksDB Option file must have a single DBOptions section"); + } + if (!has_default_cf_options_) { + return Status::Corruption( + "A RocksDB Option file must have a single CFOptions:default section"); + } + + return Status::OK(); +} + +std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line, + bool trim_only) { + size_t start = 0; + size_t end = line.size(); + + // we only support "#" style comment + if (!trim_only) { + size_t search_pos = 0; + while (search_pos < line.size()) { + size_t comment_pos = line.find('#', search_pos); + if (comment_pos == std::string::npos) { + break; + } + if (comment_pos == 0 || line[comment_pos - 1] != '\\') { + end = comment_pos; + break; + } + search_pos = comment_pos + 1; + } + } + + while (start < end && isspace(line[start]) != 0) { + ++start; + } + + // start < end implies end > 0. + while (start < end && isspace(line[end - 1]) != 0) { + --end; + } + + if (start < end) { + return line.substr(start, end - start); + } + + return ""; +} + +namespace { +bool AreEqualDoubles(const double a, const double b) { + return (fabs(a - b) < 0.00001); +} +} // namespace + +bool AreEqualOptions( + const char* opt1, const char* opt2, const OptionTypeInfo& type_info, + const std::string& opt_name, + const std::unordered_map<std::string, std::string>* opt_map) { + const char* offset1 = opt1 + type_info.offset; + const char* offset2 = opt2 + type_info.offset; + + switch (type_info.type) { + case OptionType::kBoolean: + return (*reinterpret_cast<const bool*>(offset1) == + *reinterpret_cast<const bool*>(offset2)); + case OptionType::kInt: + return (*reinterpret_cast<const int*>(offset1) == + *reinterpret_cast<const int*>(offset2)); + case OptionType::kInt32T: + return (*reinterpret_cast<const int32_t*>(offset1) == + *reinterpret_cast<const int32_t*>(offset2)); + case OptionType::kInt64T: + { + int64_t v1, v2; + GetUnaligned(reinterpret_cast<const int64_t*>(offset1), &v1); + GetUnaligned(reinterpret_cast<const int64_t*>(offset2), &v2); + return (v1 == v2); + } + case OptionType::kVectorInt: + return (*reinterpret_cast<const std::vector<int>*>(offset1) == + *reinterpret_cast<const std::vector<int>*>(offset2)); + case OptionType::kUInt: + return (*reinterpret_cast<const unsigned int*>(offset1) == + *reinterpret_cast<const unsigned int*>(offset2)); + case OptionType::kUInt32T: + return (*reinterpret_cast<const uint32_t*>(offset1) == + *reinterpret_cast<const uint32_t*>(offset2)); + case OptionType::kUInt64T: + { + uint64_t v1, v2; + GetUnaligned(reinterpret_cast<const uint64_t*>(offset1), &v1); + GetUnaligned(reinterpret_cast<const uint64_t*>(offset2), &v2); + return (v1 == v2); + } + case OptionType::kSizeT: + { + size_t v1, v2; + GetUnaligned(reinterpret_cast<const size_t*>(offset1), &v1); + GetUnaligned(reinterpret_cast<const size_t*>(offset2), &v2); + return (v1 == v2); + } + case OptionType::kString: + return (*reinterpret_cast<const std::string*>(offset1) == + *reinterpret_cast<const std::string*>(offset2)); + case OptionType::kDouble: + return AreEqualDoubles(*reinterpret_cast<const double*>(offset1), + *reinterpret_cast<const double*>(offset2)); + case OptionType::kCompactionStyle: + return (*reinterpret_cast<const CompactionStyle*>(offset1) == + *reinterpret_cast<const CompactionStyle*>(offset2)); + case OptionType::kCompactionPri: + return (*reinterpret_cast<const CompactionPri*>(offset1) == + *reinterpret_cast<const CompactionPri*>(offset2)); + case OptionType::kCompressionType: + return (*reinterpret_cast<const CompressionType*>(offset1) == + *reinterpret_cast<const CompressionType*>(offset2)); + case OptionType::kVectorCompressionType: { + const auto* vec1 = + reinterpret_cast<const std::vector<CompressionType>*>(offset1); + const auto* vec2 = + reinterpret_cast<const std::vector<CompressionType>*>(offset2); + return (*vec1 == *vec2); + } + case OptionType::kChecksumType: + return (*reinterpret_cast<const ChecksumType*>(offset1) == + *reinterpret_cast<const ChecksumType*>(offset2)); + case OptionType::kBlockBasedTableIndexType: + return ( + *reinterpret_cast<const BlockBasedTableOptions::IndexType*>( + offset1) == + *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(offset2)); + case OptionType::kBlockBasedTableDataBlockIndexType: + return ( + *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>( + offset1) == + *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>( + offset2)); + case OptionType::kWALRecoveryMode: + return (*reinterpret_cast<const WALRecoveryMode*>(offset1) == + *reinterpret_cast<const WALRecoveryMode*>(offset2)); + case OptionType::kAccessHint: + return (*reinterpret_cast<const DBOptions::AccessHint*>(offset1) == + *reinterpret_cast<const DBOptions::AccessHint*>(offset2)); + case OptionType::kInfoLogLevel: + return (*reinterpret_cast<const InfoLogLevel*>(offset1) == + *reinterpret_cast<const InfoLogLevel*>(offset2)); + case OptionType::kCompactionOptionsFIFO: { + CompactionOptionsFIFO lhs = + *reinterpret_cast<const CompactionOptionsFIFO*>(offset1); + CompactionOptionsFIFO rhs = + *reinterpret_cast<const CompactionOptionsFIFO*>(offset2); + if (lhs.max_table_files_size == rhs.max_table_files_size && + lhs.allow_compaction == rhs.allow_compaction) { + return true; + } + return false; + } + case OptionType::kCompactionOptionsUniversal: { + CompactionOptionsUniversal lhs = + *reinterpret_cast<const CompactionOptionsUniversal*>(offset1); + CompactionOptionsUniversal rhs = + *reinterpret_cast<const CompactionOptionsUniversal*>(offset2); + if (lhs.size_ratio == rhs.size_ratio && + lhs.min_merge_width == rhs.min_merge_width && + lhs.max_merge_width == rhs.max_merge_width && + lhs.max_size_amplification_percent == + rhs.max_size_amplification_percent && + lhs.compression_size_percent == rhs.compression_size_percent && + lhs.stop_style == rhs.stop_style && + lhs.allow_trivial_move == rhs.allow_trivial_move) { + return true; + } + return false; + } + default: + if (type_info.verification == OptionVerificationType::kByName || + type_info.verification == + OptionVerificationType::kByNameAllowFromNull || + type_info.verification == OptionVerificationType::kByNameAllowNull) { + std::string value1; + bool result = + SerializeSingleOptionHelper(offset1, type_info.type, &value1); + if (result == false) { + return false; + } + if (opt_map == nullptr) { + return true; + } + auto iter = opt_map->find(opt_name); + if (iter == opt_map->end()) { + return true; + } else { + if (type_info.verification == + OptionVerificationType::kByNameAllowNull) { + if (iter->second == kNullptrString || value1 == kNullptrString) { + return true; + } + } else if (type_info.verification == + OptionVerificationType::kByNameAllowFromNull) { + if (iter->second == kNullptrString) { + return true; + } + } + return (value1 == iter->second); + } + } + return false; + } +} + +Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + const DBOptions& db_opt, const std::vector<std::string>& cf_names, + const std::vector<ColumnFamilyOptions>& cf_opts, + const std::string& file_name, Env* env, + OptionsSanityCheckLevel sanity_check_level, bool ignore_unknown_options) { + RocksDBOptionsParser parser; + std::unique_ptr<SequentialFile> seq_file; + Status s = parser.Parse(file_name, env, ignore_unknown_options); + if (!s.ok()) { + return s; + } + + // Verify DBOptions + s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(), + sanity_check_level); + if (!s.ok()) { + return s; + } + + // Verify ColumnFamily Name + if (cf_names.size() != parser.cf_names()->size()) { + if (sanity_check_level >= kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionParser Error] The persisted options does not have " + "the same number of column family names as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column family " + "names than that of the specified one."); + } + } + for (size_t i = 0; i < cf_names.size(); ++i) { + if (cf_names[i] != parser.cf_names()->at(i)) { + return Status::InvalidArgument( + "[RocksDBOptionParser Error] The persisted options and the db" + "instance does not have the same name for column family ", + ToString(i)); + } + } + + // Verify Column Family Options + if (cf_opts.size() != parser.cf_opts()->size()) { + if (sanity_check_level >= kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options does not have the same number of " + "column families as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column families " + "than that of the specified number."); + } + } + for (size_t i = 0; i < cf_opts.size(); ++i) { + s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i), + &(parser.cf_opt_maps()->at(i)), sanity_check_level); + if (!s.ok()) { + return s; + } + s = VerifyTableFactory(cf_opts[i].table_factory.get(), + parser.cf_opts()->at(i).table_factory.get(), + sanity_check_level); + if (!s.ok()) { + return s; + } + } + + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyDBOptions( + const DBOptions& base_opt, const DBOptions& persisted_opt, + const std::unordered_map<std::string, std::string>* /*opt_map*/, + OptionsSanityCheckLevel sanity_check_level) { + for (auto pair : db_options_type_info) { + if (pair.second.verification == OptionVerificationType::kDeprecated) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + continue; + } + if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt), + reinterpret_cast<const char*>(&persisted_opt), + pair.second, pair.first, nullptr)) { + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + std::string base_value; + std::string persisted_value; + SerializeSingleOptionHelper( + reinterpret_cast<const char*>(&base_opt) + pair.second.offset, + pair.second.type, &base_value); + SerializeSingleOptionHelper( + reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset, + pair.second.type, &persisted_value); + snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on DBOptions::%s --- " + "The specified one is %s while the persisted one is %s.\n", + pair.first.c_str(), base_value.c_str(), + persisted_value.c_str()); + return Status::InvalidArgument(Slice(buffer, strlen(buffer))); + } + } + } + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyCFOptions( + const ColumnFamilyOptions& base_opt, + const ColumnFamilyOptions& persisted_opt, + const std::unordered_map<std::string, std::string>* persisted_opt_map, + OptionsSanityCheckLevel sanity_check_level) { + for (auto& pair : cf_options_type_info) { + if (pair.second.verification == OptionVerificationType::kDeprecated) { + // We skip checking deprecated variables as they might + // contain random values since they might not be initialized + continue; + } + if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt), + reinterpret_cast<const char*>(&persisted_opt), + pair.second, pair.first, persisted_opt_map)) { + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + std::string base_value; + std::string persisted_value; + SerializeSingleOptionHelper( + reinterpret_cast<const char*>(&base_opt) + pair.second.offset, + pair.second.type, &base_value); + SerializeSingleOptionHelper( + reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset, + pair.second.type, &persisted_value); + snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on ColumnFamilyOptions::%s --- " + "The specified one is %s while the persisted one is %s.\n", + pair.first.c_str(), base_value.c_str(), + persisted_value.c_str()); + return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); + } + } + } + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyTableFactory( + const TableFactory* base_tf, const TableFactory* file_tf, + OptionsSanityCheckLevel sanity_check_level) { + if (base_tf && file_tf) { + if (sanity_check_level > kSanityLevelNone && + std::string(base_tf->Name()) != std::string(file_tf->Name())) { + return Status::Corruption( + "[RocksDBOptionsParser]: " + "failed the verification on TableFactory->Name()"); + } + if (base_tf->Name() == BlockBasedTableFactory::kName) { + return VerifyBlockBasedTableFactory( + static_cast_with_check<const BlockBasedTableFactory, + const TableFactory>(base_tf), + static_cast_with_check<const BlockBasedTableFactory, + const TableFactory>(file_tf), + sanity_check_level); + } + // TODO(yhchiang): add checks for other table factory types + } else { + // TODO(yhchiang): further support sanity check here + } + return Status::OK(); +} +} // namespace rocksdb + +#endif // !ROCKSDB_LITE diff --git a/src/rocksdb/options/options_parser.h b/src/rocksdb/options/options_parser.h new file mode 100644 index 00000000..5aab3e7e --- /dev/null +++ b/src/rocksdb/options/options_parser.h @@ -0,0 +1,145 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include <map> +#include <string> +#include <vector> + +#include "options/options_sanity_check.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "table/block_based_table_factory.h" + +namespace rocksdb { + +#ifndef ROCKSDB_LITE + +#define ROCKSDB_OPTION_FILE_MAJOR 1 +#define ROCKSDB_OPTION_FILE_MINOR 1 + +enum OptionSection : char { + kOptionSectionVersion = 0, + kOptionSectionDBOptions, + kOptionSectionCFOptions, + kOptionSectionTableOptions, + kOptionSectionUnknown +}; + +static const std::string opt_section_titles[] = { + "Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"}; + +Status PersistRocksDBOptions(const DBOptions& db_opt, + const std::vector<std::string>& cf_names, + const std::vector<ColumnFamilyOptions>& cf_opts, + const std::string& file_name, Env* env); + +extern bool AreEqualOptions( + const char* opt1, const char* opt2, const OptionTypeInfo& type_info, + const std::string& opt_name, + const std::unordered_map<std::string, std::string>* opt_map); + +class RocksDBOptionsParser { + public: + explicit RocksDBOptionsParser(); + ~RocksDBOptionsParser() {} + void Reset(); + + Status Parse(const std::string& file_name, Env* env, + bool ignore_unknown_options = false); + static std::string TrimAndRemoveComment(const std::string& line, + const bool trim_only = false); + + const DBOptions* db_opt() const { return &db_opt_; } + const std::unordered_map<std::string, std::string>* db_opt_map() const { + return &db_opt_map_; + } + const std::vector<ColumnFamilyOptions>* cf_opts() const { return &cf_opts_; } + const std::vector<std::string>* cf_names() const { return &cf_names_; } + const std::vector<std::unordered_map<std::string, std::string>>* cf_opt_maps() + const { + return &cf_opt_maps_; + } + + const ColumnFamilyOptions* GetCFOptions(const std::string& name) { + return GetCFOptionsImpl(name); + } + size_t NumColumnFamilies() { return cf_opts_.size(); } + + static Status VerifyRocksDBOptionsFromFile( + const DBOptions& db_opt, const std::vector<std::string>& cf_names, + const std::vector<ColumnFamilyOptions>& cf_opts, + const std::string& file_name, Env* env, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch, + bool ignore_unknown_options = false); + + static Status VerifyDBOptions( + const DBOptions& base_opt, const DBOptions& new_opt, + const std::unordered_map<std::string, std::string>* new_opt_map = nullptr, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); + + static Status VerifyCFOptions( + const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt, + const std::unordered_map<std::string, std::string>* new_opt_map = nullptr, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); + + static Status VerifyTableFactory( + const TableFactory* base_tf, const TableFactory* file_tf, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); + + static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); + + protected: + bool IsSection(const std::string& line); + Status ParseSection(OptionSection* section, std::string* title, + std::string* argument, const std::string& line, + const int line_num); + + Status CheckSection(const OptionSection section, + const std::string& section_arg, const int line_num); + + Status ParseStatement(std::string* name, std::string* value, + const std::string& line, const int line_num); + + Status EndSection(const OptionSection section, const std::string& title, + const std::string& section_arg, + const std::unordered_map<std::string, std::string>& opt_map, + bool ignore_unknown_options); + + Status ValidityCheck(); + + Status InvalidArgument(const int line_num, const std::string& message); + + Status ParseVersionNumber(const std::string& ver_name, + const std::string& ver_string, const int max_count, + int* version); + + ColumnFamilyOptions* GetCFOptionsImpl(const std::string& name) { + assert(cf_names_.size() == cf_opts_.size()); + for (size_t i = 0; i < cf_names_.size(); ++i) { + if (cf_names_[i] == name) { + return &cf_opts_[i]; + } + } + return nullptr; + } + + private: + DBOptions db_opt_; + std::unordered_map<std::string, std::string> db_opt_map_; + std::vector<std::string> cf_names_; + std::vector<ColumnFamilyOptions> cf_opts_; + std::vector<std::unordered_map<std::string, std::string>> cf_opt_maps_; + bool has_version_section_; + bool has_db_options_; + bool has_default_cf_options_; + int db_version[3]; + int opt_file_version[3]; +}; + +#endif // !ROCKSDB_LITE + +} // namespace rocksdb diff --git a/src/rocksdb/options/options_sanity_check.cc b/src/rocksdb/options/options_sanity_check.cc new file mode 100644 index 00000000..d3afcc06 --- /dev/null +++ b/src/rocksdb/options/options_sanity_check.cc @@ -0,0 +1,38 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifndef ROCKSDB_LITE + +#include "options/options_sanity_check.h" + +namespace rocksdb { + +namespace { +OptionsSanityCheckLevel SanityCheckLevelHelper( + const std::unordered_map<std::string, OptionsSanityCheckLevel>& smap, + const std::string& name) { + auto iter = smap.find(name); + return iter != smap.end() ? iter->second : kSanityLevelExactMatch; +} +} + +OptionsSanityCheckLevel DBOptionSanityCheckLevel( + const std::string& option_name) { + return SanityCheckLevelHelper(sanity_level_db_options, option_name); +} + +OptionsSanityCheckLevel CFOptionSanityCheckLevel( + const std::string& option_name) { + return SanityCheckLevelHelper(sanity_level_cf_options, option_name); +} + +OptionsSanityCheckLevel BBTOptionSanityCheckLevel( + const std::string& option_name) { + return SanityCheckLevelHelper(sanity_level_bbt_options, option_name); +} + +} // namespace rocksdb + +#endif // !ROCKSDB_LITE diff --git a/src/rocksdb/options/options_sanity_check.h b/src/rocksdb/options/options_sanity_check.h new file mode 100644 index 00000000..118fdd20 --- /dev/null +++ b/src/rocksdb/options/options_sanity_check.h @@ -0,0 +1,48 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include <string> +#include <unordered_map> + +#ifndef ROCKSDB_LITE +namespace rocksdb { +// This enum defines the RocksDB options sanity level. +enum OptionsSanityCheckLevel : unsigned char { + // Performs no sanity check at all. + kSanityLevelNone = 0x00, + // Performs minimum check to ensure the RocksDB instance can be + // opened without corrupting / mis-interpreting the data. + kSanityLevelLooselyCompatible = 0x01, + // Perform exact match sanity check. + kSanityLevelExactMatch = 0xFF, +}; + +// The sanity check level for DB options +static const std::unordered_map<std::string, OptionsSanityCheckLevel> + sanity_level_db_options {}; + +// The sanity check level for column-family options +static const std::unordered_map<std::string, OptionsSanityCheckLevel> + sanity_level_cf_options = { + {"comparator", kSanityLevelLooselyCompatible}, + {"table_factory", kSanityLevelLooselyCompatible}, + {"merge_operator", kSanityLevelLooselyCompatible}}; + +// The sanity check level for block-based table options +static const std::unordered_map<std::string, OptionsSanityCheckLevel> + sanity_level_bbt_options {}; + +OptionsSanityCheckLevel DBOptionSanityCheckLevel( + const std::string& options_name); +OptionsSanityCheckLevel CFOptionSanityCheckLevel( + const std::string& options_name); +OptionsSanityCheckLevel BBTOptionSanityCheckLevel( + const std::string& options_name); + +} // namespace rocksdb + +#endif // !ROCKSDB_LITE diff --git a/src/rocksdb/options/options_settable_test.cc b/src/rocksdb/options/options_settable_test.cc new file mode 100644 index 00000000..3a6bd6a8 --- /dev/null +++ b/src/rocksdb/options/options_settable_test.cc @@ -0,0 +1,482 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include <cstring> + +#include "options/options_helper.h" +#include "rocksdb/convenience.h" +#include "util/testharness.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace rocksdb { + +// Verify options are settable from options strings. +// We take the approach that depends on compiler behavior that copy constructor +// won't touch implicit padding bytes, so that the test is fragile. +// As a result, we only run the tests to verify new fields in options are +// settable through string on limited platforms as it depends on behavior of +// compilers. +#ifndef ROCKSDB_LITE +#if defined OS_LINUX || defined OS_WIN +#ifndef __clang__ + +class OptionsSettableTest : public testing::Test { + public: + OptionsSettableTest() {} +}; + +const char kSpecialChar = 'z'; +typedef std::vector<std::pair<size_t, size_t>> OffsetGap; + +void FillWithSpecialChar(char* start_ptr, size_t total_size, + const OffsetGap& blacklist) { + size_t offset = 0; + for (auto& pair : blacklist) { + std::memset(start_ptr + offset, kSpecialChar, pair.first - offset); + offset = pair.first + pair.second; + } + std::memset(start_ptr + offset, kSpecialChar, total_size - offset); +} + +int NumUnsetBytes(char* start_ptr, size_t total_size, + const OffsetGap& blacklist) { + int total_unset_bytes_base = 0; + size_t offset = 0; + for (auto& pair : blacklist) { + for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) { + if (*ptr == kSpecialChar) { + total_unset_bytes_base++; + } + } + offset = pair.first + pair.second; + } + for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) { + if (*ptr == kSpecialChar) { + total_unset_bytes_base++; + } + } + return total_unset_bytes_base; +} + +// If the test fails, likely a new option is added to BlockBasedTableOptions +// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetBlockBasedTableOptionsFromString() and add the option to the input string +// passed to the GetBlockBasedTableOptionsFromString() in this test. +// If it is a complicated type, you also need to add the field to +// kBbtoBlacklist, and maybe add customized verification for it. +TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) { + // Items in the form of <offset, size>. Need to be in ascending order + // and not overlapping. Need to updated if new pointer-option is added. + const OffsetGap kBbtoBlacklist = { + {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), + sizeof(std::shared_ptr<FlushBlockPolicyFactory>)}, + {offsetof(struct BlockBasedTableOptions, block_cache), + sizeof(std::shared_ptr<Cache>)}, + {offsetof(struct BlockBasedTableOptions, persistent_cache), + sizeof(std::shared_ptr<PersistentCache>)}, + {offsetof(struct BlockBasedTableOptions, block_cache_compressed), + sizeof(std::shared_ptr<Cache>)}, + {offsetof(struct BlockBasedTableOptions, filter_policy), + sizeof(std::shared_ptr<const FilterPolicy>)}, + }; + + // In this test, we catch a new option of BlockBasedTableOptions that is not + // settable through GetBlockBasedTableOptionsFromString(). + // We count padding bytes of the option struct, and assert it to be the same + // as unset bytes of an option struct initialized by + // GetBlockBasedTableOptionsFromString(). + + char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *bbto = BlockBasedTableOptions(); + int unset_bytes_base = + NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); + ASSERT_GT(unset_bytes_base, 0); + bbto->~BlockBasedTableOptions(); + + // Construct the base option passed into + // GetBlockBasedTableOptionsFromString(). + bbto = new (bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); + // This option is not setable: + bbto->use_delta_encoding = true; + + char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; + BlockBasedTableOptions* new_bbto = + new (new_bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK(GetBlockBasedTableOptionsFromString( + *bbto, + "cache_index_and_filter_blocks=1;" + "cache_index_and_filter_blocks_with_high_priority=true;" + "pin_l0_filter_and_index_blocks_in_cache=1;" + "pin_top_level_index_and_filter=1;" + "index_type=kHashSearch;" + "data_block_index_type=kDataBlockBinaryAndHash;" + "data_block_hash_table_util_ratio=0.75;" + "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" + "block_cache=1M;block_cache_compressed=1k;block_size=1024;" + "block_size_deviation=8;block_restart_interval=4; " + "metadata_block_size=1024;" + "partition_filters=false;" + "index_block_restart_interval=4;" + "filter_policy=bloomfilter:4:true;whole_key_filtering=1;" + "format_version=1;" + "hash_index_allow_collision=false;" + "verify_compression=true;read_amp_bytes_per_bit=0;" + "enable_index_compression=false;" + "block_align=true", + new_bbto)); + + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoBlacklist)); + + ASSERT_TRUE(new_bbto->block_cache.get() != nullptr); + ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr); + ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr); + + bbto->~BlockBasedTableOptions(); + new_bbto->~BlockBasedTableOptions(); + + delete[] bbto_ptr; + delete[] new_bbto_ptr; +} + +// If the test fails, likely a new option is added to DBOptions +// but it cannot be set through GetDBOptionsFromString(), or the test is not +// updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetDBOptionsFromString() and add the option to the input string passed to +// DBOptionsFromString()in this test. +// If it is a complicated type, you also need to add the field to +// kDBOptionsBlacklist, and maybe add customized verification for it. +TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { + const OffsetGap kDBOptionsBlacklist = { + {offsetof(struct DBOptions, env), sizeof(Env*)}, + {offsetof(struct DBOptions, rate_limiter), + sizeof(std::shared_ptr<RateLimiter>)}, + {offsetof(struct DBOptions, sst_file_manager), + sizeof(std::shared_ptr<SstFileManager>)}, + {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)}, + {offsetof(struct DBOptions, statistics), + sizeof(std::shared_ptr<Statistics>)}, + {offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)}, + {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, wal_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, write_buffer_manager), + sizeof(std::shared_ptr<WriteBufferManager>)}, + {offsetof(struct DBOptions, listeners), + sizeof(std::vector<std::shared_ptr<EventListener>>)}, + {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)}, + {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)}, + }; + + char* options_ptr = new char[sizeof(DBOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + DBOptions* options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *options = DBOptions(); + int unset_bytes_base = + NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + ASSERT_GT(unset_bytes_base, 0); + options->~DBOptions(); + + options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + + char* new_options_ptr = new char[sizeof(DBOptions)]; + DBOptions* new_options = new (new_options_ptr) DBOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK( + GetDBOptionsFromString(*options, + "wal_bytes_per_sync=4295048118;" + "delete_obsolete_files_period_micros=4294967758;" + "WAL_ttl_seconds=4295008036;" + "WAL_size_limit_MB=4295036161;" + "wal_dir=path/to/wal_dir;" + "db_write_buffer_size=2587;" + "max_subcompactions=64330;" + "table_cache_numshardbits=28;" + "max_open_files=72;" + "max_file_opening_threads=35;" + "max_background_jobs=8;" + "base_background_compactions=3;" + "max_background_compactions=33;" + "use_fsync=true;" + "use_adaptive_mutex=false;" + "max_total_wal_size=4295005604;" + "compaction_readahead_size=0;" + "new_table_reader_for_compaction_inputs=false;" + "keep_log_file_num=4890;" + "skip_stats_update_on_db_open=false;" + "max_manifest_file_size=4295009941;" + "db_log_dir=path/to/db_log_dir;" + "skip_log_error_on_recovery=true;" + "writable_file_max_buffer_size=1048576;" + "paranoid_checks=true;" + "is_fd_close_on_exec=false;" + "bytes_per_sync=4295013613;" + "enable_thread_tracking=false;" + "recycle_log_file_num=0;" + "create_missing_column_families=true;" + "log_file_time_to_roll=3097;" + "max_background_flushes=35;" + "create_if_missing=false;" + "error_if_exists=true;" + "delayed_write_rate=4294976214;" + "manifest_preallocation_size=1222;" + "allow_mmap_writes=false;" + "stats_dump_period_sec=70127;" + "stats_persist_period_sec=54321;" + "stats_history_buffer_size=14159;" + "allow_fallocate=true;" + "allow_mmap_reads=false;" + "use_direct_reads=false;" + "use_direct_io_for_flush_and_compaction=false;" + "max_log_file_size=4607;" + "random_access_max_buffer_size=1048576;" + "advise_random_on_open=true;" + "fail_if_options_file_error=false;" + "enable_pipelined_write=false;" + "allow_concurrent_memtable_write=true;" + "wal_recovery_mode=kPointInTimeRecovery;" + "enable_write_thread_adaptive_yield=true;" + "write_thread_slow_yield_usec=5;" + "write_thread_max_yield_usec=1000;" + "access_hint_on_compaction_start=NONE;" + "info_log_level=DEBUG_LEVEL;" + "dump_malloc_stats=false;" + "allow_2pc=false;" + "avoid_flush_during_recovery=false;" + "avoid_flush_during_shutdown=false;" + "allow_ingest_behind=false;" + "preserve_deletes=false;" + "concurrent_prepare=false;" + "two_write_queues=false;" + "manual_wal_flush=false;" + "seq_per_batch=false;" + "atomic_flush=false;" + "avoid_unnecessary_blocking_io=false", + new_options)); + + ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), + kDBOptionsBlacklist)); + + options->~DBOptions(); + new_options->~DBOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; +} + +template <typename T1, typename T2> +inline int offset_of(T1 T2::*member) { + static T2 obj; + return int(size_t(&(obj.*member)) - size_t(&obj)); +} + +// If the test fails, likely a new option is added to ColumnFamilyOptions +// but it cannot be set through GetColumnFamilyOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetColumnFamilyOptionsFromString() and add the option to the input +// string passed to GetColumnFamilyOptionsFromString()in this test. +// If it is a complicated type, you also need to add the field to +// kColumnFamilyOptionsBlacklist, and maybe add customized verification +// for it. +TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { + // options in the blacklist need to appear in the same order as in + // ColumnFamilyOptions. + const OffsetGap kColumnFamilyOptionsBlacklist = { + {offset_of(&ColumnFamilyOptions::inplace_callback), + sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))}, + {offset_of( + &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), + sizeof(std::shared_ptr<const SliceTransform>)}, + {offset_of(&ColumnFamilyOptions::compression_per_level), + sizeof(std::vector<CompressionType>)}, + {offset_of( + &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional), + sizeof(std::vector<int>)}, + {offset_of(&ColumnFamilyOptions::memtable_factory), + sizeof(std::shared_ptr<MemTableRepFactory>)}, + {offset_of(&ColumnFamilyOptions::table_properties_collector_factories), + sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)}, + {offset_of(&ColumnFamilyOptions::comparator), sizeof(Comparator*)}, + {offset_of(&ColumnFamilyOptions::merge_operator), + sizeof(std::shared_ptr<MergeOperator>)}, + {offset_of(&ColumnFamilyOptions::compaction_filter), + sizeof(const CompactionFilter*)}, + {offset_of(&ColumnFamilyOptions::compaction_filter_factory), + sizeof(std::shared_ptr<CompactionFilterFactory>)}, + {offset_of(&ColumnFamilyOptions::prefix_extractor), + sizeof(std::shared_ptr<const SliceTransform>)}, + {offset_of(&ColumnFamilyOptions::table_factory), + sizeof(std::shared_ptr<TableFactory>)}, + {offset_of(&ColumnFamilyOptions::cf_paths), + sizeof(std::vector<DbPath>)}, + {offset_of(&ColumnFamilyOptions::compaction_thread_limiter), + sizeof(std::shared_ptr<ConcurrentTaskLimiter>)}, + }; + + char* options_ptr = new char[sizeof(ColumnFamilyOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *options = ColumnFamilyOptions(); + + // Deprecatd option which is not initialized. Need to set it to avoid + // Valgrind error + options->max_mem_compaction_level = 0; + + int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + ASSERT_GT(unset_bytes_base, 0); + options->~ColumnFamilyOptions(); + + options = new (options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + + // Following options are not settable through + // GetColumnFamilyOptionsFromString(): + options->rate_limit_delay_max_milliseconds = 33; + options->compaction_options_universal = CompactionOptionsUniversal(); + options->compression_opts = CompressionOptions(); + options->bottommost_compression_opts = CompressionOptions(); + options->hard_rate_limit = 0; + options->soft_rate_limit = 0; + options->purge_redundant_kvs_while_flush = false; + options->max_mem_compaction_level = 0; + options->compaction_filter = nullptr; + + char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)]; + ColumnFamilyOptions* new_options = + new (new_options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK(GetColumnFamilyOptionsFromString( + *options, + "compaction_filter_factory=mpudlojcujCompactionFilterFactory;" + "table_factory=PlainTable;" + "prefix_extractor=rocksdb.CappedPrefix.13;" + "comparator=leveldb.BytewiseComparator;" + "compression_per_level=kBZip2Compression:kBZip2Compression:" + "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:" + "kSnappyCompression;" + "max_bytes_for_level_base=986;" + "bloom_locality=8016;" + "target_file_size_base=4294976376;" + "memtable_huge_page_size=2557;" + "max_successive_merges=5497;" + "max_sequential_skip_in_iterations=4294971408;" + "arena_block_size=1893;" + "target_file_size_multiplier=35;" + "min_write_buffer_number_to_merge=9;" + "max_write_buffer_number=84;" + "write_buffer_size=1653;" + "max_compaction_bytes=64;" + "max_bytes_for_level_multiplier=60;" + "memtable_factory=SkipListFactory;" + "compression=kNoCompression;" + "bottommost_compression=kDisableCompressionOption;" + "level0_stop_writes_trigger=33;" + "num_levels=99;" + "level0_slowdown_writes_trigger=22;" + "level0_file_num_compaction_trigger=14;" + "compaction_filter=urxcqstuwnCompactionFilter;" + "soft_rate_limit=530.615385;" + "soft_pending_compaction_bytes_limit=0;" + "max_write_buffer_number_to_maintain=84;" + "merge_operator=aabcxehazrMergeOperator;" + "memtable_prefix_bloom_size_ratio=0.4642;" + "memtable_whole_key_filtering=true;" + "memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;" + "paranoid_file_checks=true;" + "force_consistency_checks=true;" + "inplace_update_num_locks=7429;" + "optimize_filters_for_hits=false;" + "level_compaction_dynamic_level_bytes=false;" + "inplace_update_support=false;" + "compaction_style=kCompactionStyleFIFO;" + "compaction_pri=kMinOverlappingRatio;" + "hard_pending_compaction_bytes_limit=0;" + "disable_auto_compactions=false;" + "report_bg_io_stats=true;" + "ttl=60;" + "sample_for_compression=0;" + "compaction_options_fifo={max_table_files_size=3;allow_" + "compaction=false;};", + new_options)); + + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist)); + + options->~ColumnFamilyOptions(); + new_options->~ColumnFamilyOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; +} +#endif // !__clang__ +#endif // OS_LINUX || OS_WIN +#endif // !ROCKSDB_LITE + +} // namespace rocksdb + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/src/rocksdb/options/options_test.cc b/src/rocksdb/options/options_test.cc new file mode 100644 index 00000000..586e5697 --- /dev/null +++ b/src/rocksdb/options/options_test.cc @@ -0,0 +1,1908 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include <cctype> +#include <cstring> +#include <unordered_map> +#include <inttypes.h> + +#include "cache/lru_cache.h" +#include "cache/sharded_cache.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "options/options_sanity_check.h" +#include "port/port.h" +#include "rocksdb/cache.h" +#include "rocksdb/convenience.h" +#include "rocksdb/memtablerep.h" +#include "rocksdb/utilities/leveldb_options.h" +#include "rocksdb/utilities/object_registry.h" +#include "util/random.h" +#include "util/stderr_logger.h" +#include "util/string_util.h" +#include "util/testharness.h" +#include "util/testutil.h" +#include "utilities/merge_operators/bytesxor.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace rocksdb { + +class OptionsTest : public testing::Test {}; + +#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE +TEST_F(OptionsTest, GetOptionsFromMapTest) { + std::unordered_map<std::string, std::string> cf_options_map = { + {"write_buffer_size", "1"}, + {"max_write_buffer_number", "2"}, + {"min_write_buffer_number_to_merge", "3"}, + {"max_write_buffer_number_to_maintain", "99"}, + {"compression", "kSnappyCompression"}, + {"compression_per_level", + "kNoCompression:" + "kSnappyCompression:" + "kZlibCompression:" + "kBZip2Compression:" + "kLZ4Compression:" + "kLZ4HCCompression:" + "kXpressCompression:" + "kZSTD:" + "kZSTDNotFinalCompression"}, + {"bottommost_compression", "kLZ4Compression"}, + {"bottommost_compression_opts", "5:6:7:8:9:true"}, + {"compression_opts", "4:5:6:7:8:true"}, + {"num_levels", "8"}, + {"level0_file_num_compaction_trigger", "8"}, + {"level0_slowdown_writes_trigger", "9"}, + {"level0_stop_writes_trigger", "10"}, + {"target_file_size_base", "12"}, + {"target_file_size_multiplier", "13"}, + {"max_bytes_for_level_base", "14"}, + {"level_compaction_dynamic_level_bytes", "true"}, + {"max_bytes_for_level_multiplier", "15.0"}, + {"max_bytes_for_level_multiplier_additional", "16:17:18"}, + {"max_compaction_bytes", "21"}, + {"soft_rate_limit", "1.1"}, + {"hard_rate_limit", "2.1"}, + {"hard_pending_compaction_bytes_limit", "211"}, + {"arena_block_size", "22"}, + {"disable_auto_compactions", "true"}, + {"compaction_style", "kCompactionStyleLevel"}, + {"compaction_pri", "kOldestSmallestSeqFirst"}, + {"verify_checksums_in_compaction", "false"}, + {"compaction_options_fifo", "23"}, + {"max_sequential_skip_in_iterations", "24"}, + {"inplace_update_support", "true"}, + {"report_bg_io_stats", "true"}, + {"compaction_measure_io_stats", "false"}, + {"inplace_update_num_locks", "25"}, + {"memtable_prefix_bloom_size_ratio", "0.26"}, + {"memtable_whole_key_filtering", "true"}, + {"memtable_huge_page_size", "28"}, + {"bloom_locality", "29"}, + {"max_successive_merges", "30"}, + {"min_partial_merge_operands", "31"}, + {"prefix_extractor", "fixed:31"}, + {"optimize_filters_for_hits", "true"}, + }; + + std::unordered_map<std::string, std::string> db_options_map = { + {"create_if_missing", "false"}, + {"create_missing_column_families", "true"}, + {"error_if_exists", "false"}, + {"paranoid_checks", "true"}, + {"max_open_files", "32"}, + {"max_total_wal_size", "33"}, + {"use_fsync", "true"}, + {"db_log_dir", "/db_log_dir"}, + {"wal_dir", "/wal_dir"}, + {"delete_obsolete_files_period_micros", "34"}, + {"max_background_compactions", "35"}, + {"max_background_flushes", "36"}, + {"max_log_file_size", "37"}, + {"log_file_time_to_roll", "38"}, + {"keep_log_file_num", "39"}, + {"recycle_log_file_num", "5"}, + {"max_manifest_file_size", "40"}, + {"table_cache_numshardbits", "41"}, + {"WAL_ttl_seconds", "43"}, + {"WAL_size_limit_MB", "44"}, + {"manifest_preallocation_size", "45"}, + {"allow_mmap_reads", "true"}, + {"allow_mmap_writes", "false"}, + {"use_direct_reads", "false"}, + {"use_direct_io_for_flush_and_compaction", "false"}, + {"is_fd_close_on_exec", "true"}, + {"skip_log_error_on_recovery", "false"}, + {"stats_dump_period_sec", "46"}, + {"stats_persist_period_sec", "57"}, + {"stats_history_buffer_size", "69"}, + {"advise_random_on_open", "true"}, + {"use_adaptive_mutex", "false"}, + {"new_table_reader_for_compaction_inputs", "true"}, + {"compaction_readahead_size", "100"}, + {"random_access_max_buffer_size", "3145728"}, + {"writable_file_max_buffer_size", "314159"}, + {"bytes_per_sync", "47"}, + {"wal_bytes_per_sync", "48"}, + }; + + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + ASSERT_OK(GetColumnFamilyOptionsFromMap( + base_cf_opt, cf_options_map, &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 1U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2); + ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3); + ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99); + ASSERT_EQ(new_cf_opt.compression, kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U); + ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression); + ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression); + ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD); + ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression); + ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4); + ASSERT_EQ(new_cf_opt.compression_opts.level, 5); + ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6); + ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7); + ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8); + ASSERT_EQ(new_cf_opt.compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9); + ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true); + ASSERT_EQ(new_cf_opt.num_levels, 8); + ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8); + ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9); + ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10); + ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12)); + ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U); + ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17); + ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18); + ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21); + ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211); + ASSERT_EQ(new_cf_opt.arena_block_size, 22U); + ASSERT_EQ(new_cf_opt.disable_auto_compactions, true); + ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel); + ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst); + ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size, + static_cast<uint64_t>(23)); + ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations, + static_cast<uint64_t>(24)); + ASSERT_EQ(new_cf_opt.inplace_update_support, true); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U); + ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26); + ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true); + ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U); + ASSERT_EQ(new_cf_opt.bloom_locality, 29U); + ASSERT_EQ(new_cf_opt.max_successive_merges, 30U); + ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr); + ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true); + ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()), + "rocksdb.FixedPrefix.31"); + + cf_options_map["write_buffer_size"] = "hello"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap( + base_cf_opt, cf_options_map, &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + cf_options_map["write_buffer_size"] = "1"; + ASSERT_OK(GetColumnFamilyOptionsFromMap( + base_cf_opt, cf_options_map, &new_cf_opt)); + + cf_options_map["unknown_option"] = "1"; + ASSERT_NOK(GetColumnFamilyOptionsFromMap( + base_cf_opt, cf_options_map, &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, + &new_cf_opt, + false, /* input_strings_escaped */ + true /* ignore_unknown_options */)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */ + kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/)); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */ + kSanityLevelExactMatch /* default for VerifyCFOptions */)); + + DBOptions base_db_opt; + DBOptions new_db_opt; + ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); + ASSERT_EQ(new_db_opt.create_if_missing, false); + ASSERT_EQ(new_db_opt.create_missing_column_families, true); + ASSERT_EQ(new_db_opt.error_if_exists, false); + ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.max_open_files, 32); + ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33)); + ASSERT_EQ(new_db_opt.use_fsync, true); + ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir"); + ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir"); + ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros, + static_cast<uint64_t>(34)); + ASSERT_EQ(new_db_opt.max_background_compactions, 35); + ASSERT_EQ(new_db_opt.max_background_flushes, 36); + ASSERT_EQ(new_db_opt.max_log_file_size, 37U); + ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U); + ASSERT_EQ(new_db_opt.keep_log_file_num, 39U); + ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U); + ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40)); + ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41); + ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43)); + ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44)); + ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U); + ASSERT_EQ(new_db_opt.allow_mmap_reads, true); + ASSERT_EQ(new_db_opt.allow_mmap_writes, false); + ASSERT_EQ(new_db_opt.use_direct_reads, false); + ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false); + ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true); + ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false); + ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U); + ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U); + ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U); + ASSERT_EQ(new_db_opt.advise_random_on_open, true); + ASSERT_EQ(new_db_opt.use_adaptive_mutex, false); + ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true); + ASSERT_EQ(new_db_opt.compaction_readahead_size, 100); + ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728); + ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159); + ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47)); + ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48)); + + db_options_map["max_open_files"] = "hello"; + ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( + base_db_opt, new_db_opt, nullptr, /* new_opt_map */ + kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */)); + + // unknow options should fail parsing without ignore_unknown_options = true + db_options_map["unknown_db_option"] = "1"; + ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt)); + + ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt, + false, /* input_strings_escaped */ + true /* ignore_unknown_options */)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions( + base_db_opt, new_db_opt, nullptr, /* new_opt_map */ + kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */)); + ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions( + base_db_opt, new_db_opt, nullptr, /* new_opt_mat */ + kSanityLevelExactMatch /* default for VerifyDBOptions */)); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in + // ROCKSDB_LITE +TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { + ColumnFamilyOptions base_cf_opt; + ColumnFamilyOptions new_cf_opt; + base_cf_opt.table_factory.reset(); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=5", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 5U); + ASSERT_TRUE(new_cf_opt.table_factory == nullptr); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=6;", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 6U); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + " write_buffer_size = 7 ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 7U); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + " write_buffer_size = 8 ; ", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 8U); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 9U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=11; max_write_buffer_number = 12 ;", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 11U); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12); + // Wrong name "max_write_buffer_number_" + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=13;max_write_buffer_number_=14;", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + // Comparator from object registry + std::string kCompName = "reverse_comp"; + static Registrar<const Comparator> test_reg_a( + kCompName, [](const std::string& /*name*/, + std::unique_ptr<const Comparator>* /*comparator_guard*/) { + return ReverseBytewiseComparator(); + }); + + ASSERT_OK(GetColumnFamilyOptionsFromString( + base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator()); + + // MergeOperator from object registry + std::unique_ptr<BytesXOROperator> bxo(new BytesXOROperator()); + std::string kMoName = bxo->Name(); + static Registrar<std::shared_ptr<MergeOperator>> test_reg_b( + kMoName, [](const std::string& /*name*/, + std::unique_ptr<std::shared_ptr<MergeOperator>>* + merge_operator_guard) { + merge_operator_guard->reset( + new std::shared_ptr<MergeOperator>(new BytesXOROperator())); + return merge_operator_guard->get(); + }); + + ASSERT_OK(GetColumnFamilyOptionsFromString( + base_cf_opt, "merge_operator=" + kMoName + ";", &new_cf_opt)); + ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name())); + + // Wrong key/value pair + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + // Error Paring value + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + // Missing option name + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=13; =100;", &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + const int64_t kilo = 1024UL; + const int64_t mega = 1024 * kilo; + const int64_t giga = 1024 * mega; + const int64_t tera = 1024 * giga; + + // Units (k) + ASSERT_OK(GetColumnFamilyOptionsFromString( + base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo); + // Units (m) + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "max_write_buffer_number=16m;inplace_update_num_locks=17M", + &new_cf_opt)); + ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega); + ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17 * mega); + // Units (g) + ASSERT_OK(GetColumnFamilyOptionsFromString( + base_cf_opt, + "write_buffer_size=18g;prefix_extractor=capped:8;" + "arena_block_size=19G", + &new_cf_opt)); + + ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga); + ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga); + ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr); + std::string prefix_name(new_cf_opt.prefix_extractor->Name()); + ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8"); + + // Units (t) + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt)); + ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera); + ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera); + + // Nested block based table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Last one + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;}", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + // Mismatch curly braces + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={{{block_size=4;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + // Unexpected chars after closing curly brace + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa;" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_size=4;}xdfa", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + // Invalid block based table option + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={xx_block_size=4;}", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "optimize_filters_for_hits=true", + &new_cf_opt)); + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "optimize_filters_for_hits=false", + &new_cf_opt)); + + ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, + "optimize_filters_for_hits=junk", + &new_cf_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt)); + + // Nested plain table options + // Empty + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + + // memtable factory + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "memtable=skip_list:10;arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported +TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { + BlockBasedTableOptions table_opt; + BlockBasedTableOptions new_opt; + // make sure default values are overwritten by something else + ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, + "cache_index_and_filter_blocks=1;index_type=kHashSearch;" + "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" + "block_cache=1M;block_cache_compressed=1k;block_size=1024;" + "block_size_deviation=8;block_restart_interval=4;" + "filter_policy=bloomfilter:4:true;whole_key_filtering=1;", + &new_opt)); + ASSERT_TRUE(new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch); + ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash); + ASSERT_TRUE(new_opt.hash_index_allow_collision); + ASSERT_TRUE(new_opt.no_block_cache); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL); + ASSERT_EQ(new_opt.block_size, 1024UL); + ASSERT_EQ(new_opt.block_size_deviation, 8); + ASSERT_EQ(new_opt.block_restart_interval, 4); + ASSERT_TRUE(new_opt.filter_policy != nullptr); + + // unknown option + ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearch;" + "bad_option=1", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized index type + ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, + "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized checksum type + ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, + "cache_index_and_filter_blocks=1;checksum=kxxHashXX", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.index_type, new_opt.index_type); + + // unrecognized filter policy name + ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, + "cache_index_and_filter_blocks=1;" + "filter_policy=bloomfilterxx:4:true", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy); + + // unrecognized filter policy config + ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt, + "cache_index_and_filter_blocks=1;" + "filter_policy=bloomfilter:4", + &new_opt)); + ASSERT_EQ(table_opt.cache_index_and_filter_blocks, + new_opt.cache_index_and_filter_blocks); + ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy); + + // Check block cache options are overwritten when specified + // in new format as a struct. + ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache_compressed)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache_compressed)->GetHighPriPoolRatio(), + 0.5); + + // Set only block cache capacity. Check other values are + // reset to default values. + ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, + "block_cache={capacity=2M};" + "block_cache_compressed={capacity=2M}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL); + // Default values + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache)->GetNumShardBits(), + GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity())); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.0); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL); + // Default values + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache_compressed)->GetNumShardBits(), + GetDefaultCacheShardBits( + new_opt.block_cache_compressed->GetCapacity())); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache_compressed)->GetHighPriPoolRatio(), + 0.0); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, + "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};" + "block_cache_compressed={num_shard_bits=5;" + "high_pri_pool_ratio=0.5;}", + &new_opt)); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0); + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache)->GetNumShardBits(), 5); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.5); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0); + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache_compressed)->GetNumShardBits(), 5); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache_compressed)->GetHighPriPoolRatio(), + 0.5); + + // Set couple of block cache options. + ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt, + "block_cache={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;};" + "block_cache_compressed={capacity=1M;num_shard_bits=4;" + "strict_capacity_limit=true;}", + &new_opt)); + ASSERT_TRUE(new_opt.block_cache != nullptr); + ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache)->GetHighPriPoolRatio(), 0.0); + ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); + ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL); + ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>( + new_opt.block_cache_compressed)->GetNumShardBits(), 4); + ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true); + ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>( + new_opt.block_cache_compressed)->GetHighPriPoolRatio(), + 0.0); +} +#endif // !ROCKSDB_LITE + + +#ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported +TEST_F(OptionsTest, GetPlainTableOptionsFromString) { + PlainTableOptions table_opt; + PlainTableOptions new_opt; + // make sure default values are overwritten by something else + ASSERT_OK(GetPlainTableOptionsFromString(table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;" + "full_scan_mode=true;store_index_in_file=true", + &new_opt)); + ASSERT_EQ(new_opt.user_key_len, 66); + ASSERT_EQ(new_opt.bloom_bits_per_key, 20); + ASSERT_EQ(new_opt.hash_table_ratio, 0.5); + ASSERT_EQ(new_opt.index_sparseness, 8); + ASSERT_EQ(new_opt.huge_page_tlb_size, 4); + ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix); + ASSERT_TRUE(new_opt.full_scan_mode); + ASSERT_TRUE(new_opt.store_index_in_file); + + // unknown option + ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "bad_option=1", + &new_opt)); + + // unrecognized EncodingType + ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "encoding_type=kPrefixXX", + &new_opt)); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported +TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { + std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr; + + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000", + &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist", + &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000", + &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt", + &new_mem_factory)); + + ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory)); + // CuckooHash memtable is already removed. + ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory)); + + ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory)); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite +TEST_F(OptionsTest, GetOptionsFromStringTest) { + Options base_options, new_options; + base_options.write_buffer_size = 20; + base_options.min_write_buffer_number_to_merge = 15; + BlockBasedTableOptions block_based_table_options; + block_based_table_options.cache_index_and_filter_blocks = true; + base_options.table_factory.reset( + NewBlockBasedTableFactory(block_based_table_options)); + ASSERT_OK(GetOptionsFromString( + base_options, + "write_buffer_size=10;max_write_buffer_number=16;" + "block_based_table_factory={block_cache=1M;block_size=4;};" + "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;" + "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files=" + "1;" + "rate_limiter_bytes_per_sec=1024", + &new_options)); + + ASSERT_EQ(new_options.compression_opts.window_bits, 4); + ASSERT_EQ(new_options.compression_opts.level, 5); + ASSERT_EQ(new_options.compression_opts.strategy, 6); + ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0); + ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0); + ASSERT_EQ(new_options.compression_opts.enabled, false); + ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption); + ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5); + ASSERT_EQ(new_options.bottommost_compression_opts.level, 6); + ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7); + ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0); + ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0); + ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); + ASSERT_EQ(new_options.write_buffer_size, 10U); + ASSERT_EQ(new_options.max_write_buffer_number, 16); + BlockBasedTableOptions new_block_based_table_options = + dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get()) + ->table_options(); + ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20); + ASSERT_EQ(new_block_based_table_options.block_size, 4U); + // don't overwrite block based table options + ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks); + + ASSERT_EQ(new_options.create_if_missing, true); + ASSERT_EQ(new_options.max_open_files, 1); + ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); +} + +TEST_F(OptionsTest, DBOptionsSerialization) { + Options base_options, new_options; + Random rnd(301); + + // Phase 1: Make big change in base_options + test::RandomInitDBOptions(&base_options, &rnd); + + // Phase 2: obtain a string from base_option + std::string base_options_file_content; + ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options)); + + // Phase 3: Set new_options from the derived string and expect + // new_options == base_options + ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content, + &new_options)); + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options)); +} + +TEST_F(OptionsTest, OptionsComposeDecompose) { + // build an Options from DBOptions + CFOptions, then decompose it to verify + // we get same constituent options. + DBOptions base_db_opts; + ColumnFamilyOptions base_cf_opts; + + Random rnd(301); + test::RandomInitDBOptions(&base_db_opts, &rnd); + test::RandomInitCFOptions(&base_cf_opts, &rnd); + + Options base_opts(base_db_opts, base_cf_opts); + DBOptions new_db_opts(base_opts); + ColumnFamilyOptions new_cf_opts(base_opts); + + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opts, new_db_opts)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opts, new_cf_opts)); + delete new_cf_opts.compaction_filter; +} + +TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) { + ColumnFamilyOptions base_opt, new_opt; + Random rnd(302); + // Phase 1: randomly assign base_opt + // custom type options + test::RandomInitCFOptions(&base_opt, &rnd); + + // Phase 2: obtain a string from base_opt + std::string base_options_file_content; + ASSERT_OK( + GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt)); + + // Phase 3: Set new_opt from the derived string and expect + // new_opt == base_opt + ASSERT_OK(GetColumnFamilyOptionsFromString( + ColumnFamilyOptions(), base_options_file_content, &new_opt)); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt)); + if (base_opt.compaction_filter) { + delete base_opt.compaction_filter; + } +} + +#endif // !ROCKSDB_LITE + +Status StringToMap( + const std::string& opts_str, + std::unordered_map<std::string, std::string>* opts_map); + +#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE +TEST_F(OptionsTest, StringToMapTest) { + std::unordered_map<std::string, std::string> opts_map; + // Regular options + ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "v2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Value with '=' + opts_map.clear(); + ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map)); + ASSERT_EQ(opts_map["k1"], "=v1"); + ASSERT_EQ(opts_map["k2"], "v2="); + // Overwrriten option + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Empty value + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_EQ(opts_map["k3"], "v3"); + ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); + ASSERT_EQ(opts_map["k4"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_EQ(opts_map["k3"], "v3"); + ASSERT_TRUE(opts_map.find("k4") != opts_map.end()); + ASSERT_EQ(opts_map["k4"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); + ASSERT_EQ(opts_map["k3"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_TRUE(opts_map.find("k2") != opts_map.end()); + ASSERT_EQ(opts_map["k2"], ""); + ASSERT_TRUE(opts_map.find("k3") != opts_map.end()); + ASSERT_EQ(opts_map["k3"], ""); + // Regular nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2"); + ASSERT_EQ(opts_map["k3"], "v3"); + // Multi-level nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};" + "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}"); + ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}"); + ASSERT_EQ(opts_map["k4"], "v4"); + // Garbage inside curly braces + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "dfad="); + ASSERT_EQ(opts_map["k3"], "="); + ASSERT_EQ(opts_map["k4"], "v4"); + // Empty nested options + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], ""); + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}"); + // With random spaces + opts_map.clear(); + ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; " + "k3={ { } }; k4= v4 ", + &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}"); + ASSERT_EQ(opts_map["k3"], "{ }"); + ASSERT_EQ(opts_map["k4"], "v4"); + + // Empty key + ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map)); + ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map)); + // Mismatch curly braces + ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map)); + // However this is valid! + opts_map.clear(); + ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map)); + ASSERT_EQ(opts_map["k1"], "v1"); + ASSERT_EQ(opts_map["k2"], "}"); + ASSERT_EQ(opts_map["k3"], "v3"); + + // Invalid chars after closing curly brace + ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map)); + ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map)); +} +#endif // ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE +TEST_F(OptionsTest, StringToMapRandomTest) { + std::unordered_map<std::string, std::string> opts_map; + // Make sure segfault is not hit by semi-random strings + + std::vector<std::string> bases = { + "a={aa={};tt={xxx={}}};c=defff", + "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}", + "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"}; + + for (std::string base : bases) { + for (int rand_seed = 301; rand_seed < 401; rand_seed++) { + Random rnd(rand_seed); + for (int attempt = 0; attempt < 10; attempt++) { + std::string str = base; + // Replace random position to space + size_t pos = static_cast<size_t>( + rnd.Uniform(static_cast<int>(base.size()))); + str[pos] = ' '; + Status s = StringToMap(str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + opts_map.clear(); + } + } + } + + // Random Construct a string + std::vector<char> chars = {'{', '}', ' ', '=', ';', 'c'}; + for (int rand_seed = 301; rand_seed < 1301; rand_seed++) { + Random rnd(rand_seed); + int len = rnd.Uniform(30); + std::string str = ""; + for (int attempt = 0; attempt < len; attempt++) { + // Add a random character + size_t pos = static_cast<size_t>( + rnd.Uniform(static_cast<int>(chars.size()))); + str.append(1, chars[pos]); + } + Status s = StringToMap(str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + s = StringToMap("name=" + str, &opts_map); + ASSERT_TRUE(s.ok() || s.IsInvalidArgument()); + opts_map.clear(); + } +} + +TEST_F(OptionsTest, GetStringFromCompressionType) { + std::string res; + + ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression)); + ASSERT_EQ(res, "kNoCompression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression)); + ASSERT_EQ(res, "kSnappyCompression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption)); + ASSERT_EQ(res, "kDisableCompressionOption"); + + ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression)); + ASSERT_EQ(res, "kLZ4Compression"); + + ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression)); + ASSERT_EQ(res, "kZlibCompression"); + + ASSERT_NOK( + GetStringFromCompressionType(&res, static_cast<CompressionType>(-10))); +} +#endif // !ROCKSDB_LITE + +TEST_F(OptionsTest, ConvertOptionsTest) { + LevelDBOptions leveldb_opt; + Options converted_opt = ConvertOptions(leveldb_opt); + + ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing); + ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists); + ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks); + ASSERT_EQ(converted_opt.env, leveldb_opt.env); + ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log); + ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size); + ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); + ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); + + std::shared_ptr<TableFactory> tb_guard = converted_opt.table_factory; + BlockBasedTableFactory* table_factory = + dynamic_cast<BlockBasedTableFactory*>(converted_opt.table_factory.get()); + + ASSERT_TRUE(table_factory != nullptr); + + const BlockBasedTableOptions table_opt = table_factory->table_options(); + + ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20); + ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size); + ASSERT_EQ(table_opt.block_restart_interval, + leveldb_opt.block_restart_interval); + ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy); +} + +#ifndef ROCKSDB_LITE +class OptionsParserTest : public testing::Test { + public: + OptionsParserTest() { env_.reset(new test::StringEnv(Env::Default())); } + + protected: + std::unique_ptr<test::StringEnv> env_; +}; + +TEST_F(OptionsParserTest, Comment) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[ DBOptions ]\n" + " # note that we don't support space around \"=\"\n" + " max_open_files=12345;\n" + " max_background_flushes=301 # comment after a statement is fine\n" + " # max_background_flushes=1000 # this line would be ignored\n" + " # max_background_compactions=2000 # so does this one\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"default\"] # column family must be specified\n" + " # in the correct order\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_OK(parser.Parse(kTestFileName, env_.get())); + + ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt)); + ASSERT_EQ(parser.NumColumnFamilies(), 1U); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + *parser.GetCFOptions("default"), cf_opt)); +} + +TEST_F(OptionsParserTest, ExtraSpace) { + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[ Version ]\n" + " rocksdb_version = 3.14.0 \n" + " options_file_version=1 # some comment\n" + "[DBOptions ] # some comment\n" + "max_open_files=12345 \n" + " max_background_flushes = 301 \n" + " max_total_wal_size = 1024 # keep_log_file_num=1000\n" + " [CFOptions \"default\" ]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_OK(parser.Parse(kTestFileName, env_.get())); +} + +TEST_F(OptionsParserTest, MissingDBOptions) { + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[CFOptions \"default\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); +} + +TEST_F(OptionsParserTest, DoubleDBOptions) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[DBOptions]\n" + "[CFOptions \"default\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); +} + +TEST_F(OptionsParserTest, NoDefaultCFOptions) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"something_else\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); +} + +TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"something_else\"]\n" + " # if a section is blank, we will use the default\n" + "[CFOptions \"default\"]\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); +} + +TEST_F(OptionsParserTest, DuplicateCFOptions) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.14.0\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + "[CFOptions \"default\"]\n" + "[CFOptions \"something_else\"]\n" + "[CFOptions \"something_else\"]\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); +} + +TEST_F(OptionsParserTest, IgnoreUnknownOptions) { + for (int case_id = 0; case_id < 5; case_id++) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string version_string; + bool should_ignore = true; + if (case_id == 0) { + // same version + should_ignore = false; + version_string = + ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) + ".0"; + } else if (case_id == 1) { + // higher minor version + should_ignore = true; + version_string = + ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR + 1) + ".0"; + } else if (case_id == 2) { + // higher major version. + should_ignore = true; + version_string = ToString(ROCKSDB_MAJOR + 1) + ".0.0"; + } else if (case_id == 3) { + // lower minor version +#if ROCKSDB_MINOR == 0 + continue; +#else + version_string = + ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR - 1) + ".0"; + should_ignore = false; +#endif + } else { + // lower major version + should_ignore = false; + version_string = + ToString(ROCKSDB_MAJOR - 1) + "." + ToString(ROCKSDB_MINOR) + ".0"; + } + + std::string options_file_content = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=" + + version_string + + "\n" + " options_file_version=1\n" + "[DBOptions]\n" + " max_open_files=12345\n" + " max_background_flushes=301\n" + " max_total_wal_size=1024 # keep_log_file_num=1000\n" + " unknown_db_option1=321\n" + " unknown_db_option2=false\n" + "[CFOptions \"default\"]\n" + " unknown_cf_option1=hello\n" + "[CFOptions \"something_else\"]\n" + " unknown_cf_option2=world\n" + " # if a section is blank, we will use the default\n"; + + const std::string kTestFileName = "test-rocksdb-options.ini"; + env_->DeleteFile(kTestFileName); + env_->WriteToNewFile(kTestFileName, options_file_content); + RocksDBOptionsParser parser; + ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); + if (should_ignore) { + ASSERT_OK(parser.Parse(kTestFileName, env_.get(), + true /* ignore_unknown_options */)); + } else { + ASSERT_NOK(parser.Parse(kTestFileName, env_.get(), + true /* ignore_unknown_options */)); + } + } +} + +TEST_F(OptionsParserTest, ParseVersion) { + DBOptions db_opt; + db_opt.max_open_files = 12345; + db_opt.max_background_flushes = 301; + db_opt.max_total_wal_size = 1024; + ColumnFamilyOptions cf_opt; + + std::string file_template = + "# This is a testing option string.\n" + "# Currently we only support \"#\" styled comment.\n" + "\n" + "[Version]\n" + " rocksdb_version=3.13.1\n" + " options_file_version=%s\n" + "[DBOptions]\n" + "[CFOptions \"default\"]\n"; + const int kLength = 1000; + char buffer[kLength]; + RocksDBOptionsParser parser; + + const std::vector<std::string> invalid_versions = { + "a.b.c", "3.2.2b", "3.-12", "3. 1", // only digits and dots are allowed + "1.2.3.4", + "1.2.3" // can only contains at most one dot. + "0", // options_file_version must be at least one + "3..2", + ".", ".1.2", // must have at least one digit before each dot + "1.2.", "1.", "2.34."}; // must have at least one digit after each dot + for (auto iv : invalid_versions) { + snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str()); + + parser.Reset(); + env_->WriteToNewFile(iv, buffer); + ASSERT_NOK(parser.Parse(iv, env_.get())); + } + + const std::vector<std::string> valid_versions = { + "1.232", "100", "3.12", "1", "12.3 ", " 1.25 "}; + for (auto vv : valid_versions) { + snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str()); + parser.Reset(); + env_->WriteToNewFile(vv, buffer); + ASSERT_OK(parser.Parse(vv, env_.get())); + } +} + +void VerifyCFPointerTypedOptions( + ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt, + const std::unordered_map<std::string, std::string>* new_cf_opt_map) { + std::string name_buffer; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, + new_cf_opt_map)); + + // change the name of merge operator back-and-forth + { + auto* merge_operator = dynamic_cast<test::ChanglingMergeOperator*>( + base_cf_opt->merge_operator.get()); + if (merge_operator != nullptr) { + name_buffer = merge_operator->Name(); + // change the name and expect non-ok status + merge_operator->SetName("some-other-name"); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // change the name back and expect ok status + merge_operator->SetName(name_buffer); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, + new_cf_opt_map)); + } + } + + // change the name of the compaction filter factory back-and-forth + { + auto* compaction_filter_factory = + dynamic_cast<test::ChanglingCompactionFilterFactory*>( + base_cf_opt->compaction_filter_factory.get()); + if (compaction_filter_factory != nullptr) { + name_buffer = compaction_filter_factory->Name(); + // change the name and expect non-ok status + compaction_filter_factory->SetName("some-other-name"); + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // change the name back and expect ok status + compaction_filter_factory->SetName(name_buffer); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, + new_cf_opt_map)); + } + } + + // test by setting compaction_filter to nullptr + { + auto* tmp_compaction_filter = base_cf_opt->compaction_filter; + if (tmp_compaction_filter != nullptr) { + base_cf_opt->compaction_filter = nullptr; + // set compaction_filter to nullptr and expect non-ok status + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // set the value back and expect ok status + base_cf_opt->compaction_filter = tmp_compaction_filter; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, + new_cf_opt_map)); + } + } + + // test by setting table_factory to nullptr + { + auto tmp_table_factory = base_cf_opt->table_factory; + if (tmp_table_factory != nullptr) { + base_cf_opt->table_factory.reset(); + // set table_factory to nullptr and expect non-ok status + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // set the value back and expect ok status + base_cf_opt->table_factory = tmp_table_factory; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, + new_cf_opt_map)); + } + } + + // test by setting memtable_factory to nullptr + { + auto tmp_memtable_factory = base_cf_opt->memtable_factory; + if (tmp_memtable_factory != nullptr) { + base_cf_opt->memtable_factory.reset(); + // set memtable_factory to nullptr and expect non-ok status + ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( + *base_cf_opt, *new_cf_opt, new_cf_opt_map)); + // set the value back and expect ok status + base_cf_opt->memtable_factory = tmp_memtable_factory; + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt, + new_cf_opt_map)); + } + } +} + +TEST_F(OptionsParserTest, DumpAndParse) { + DBOptions base_db_opt; + std::vector<ColumnFamilyOptions> base_cf_opts; + std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3", + "c:f:4:4:4" + "p\\i\\k\\a\\chu\\\\\\", + "###rocksdb#1-testcf#2###"}; + const int num_cf = static_cast<int>(cf_names.size()); + Random rnd(302); + test::RandomInitDBOptions(&base_db_opt, &rnd); + base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG"; + + BlockBasedTableOptions special_bbto; + special_bbto.cache_index_and_filter_blocks = true; + special_bbto.block_size = 999999; + + for (int c = 0; c < num_cf; ++c) { + ColumnFamilyOptions cf_opt; + Random cf_rnd(0xFB + c); + test::RandomInitCFOptions(&cf_opt, &cf_rnd); + if (c < 4) { + cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c)); + } + if (c < 3) { + cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c)); + } else if (c == 4) { + cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto)); + } + base_cf_opts.emplace_back(cf_opt); + } + + const std::string kOptionsFileName = "test-persisted-options.ini"; + ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts, + kOptionsFileName, env_.get())); + + RocksDBOptionsParser parser; + ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); + + // Make sure block-based table factory options was deserialized correctly + std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory; + ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name())); + const BlockBasedTableOptions& parsed_bbto = + static_cast<BlockBasedTableFactory*>(ttf.get())->table_options(); + ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size); + ASSERT_EQ(special_bbto.cache_index_and_filter_blocks, + parsed_bbto.cache_index_and_filter_blocks); + + ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get())); + + ASSERT_OK( + RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt)); + for (int c = 0; c < num_cf; ++c) { + const auto* cf_opt = parser.GetCFOptions(cf_names[c]); + ASSERT_NE(cf_opt, nullptr); + ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( + base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c)))); + } + + // Further verify pointer-typed options + for (int c = 0; c < num_cf; ++c) { + const auto* cf_opt = parser.GetCFOptions(cf_names[c]); + ASSERT_NE(cf_opt, nullptr); + VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt, + &(parser.cf_opt_maps()->at(c))); + } + + ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr); + + base_db_opt.max_open_files++; + ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get())); + + for (int c = 0; c < num_cf; ++c) { + if (base_cf_opts[c].compaction_filter) { + delete base_cf_opts[c].compaction_filter; + } + } +} + +TEST_F(OptionsParserTest, DifferentDefault) { + const std::string kOptionsFileName = "test-persisted-options.ini"; + + ColumnFamilyOptions cf_level_opts; + ASSERT_EQ(CompactionPri::kMinOverlappingRatio, cf_level_opts.compaction_pri); + cf_level_opts.OptimizeLevelStyleCompaction(); + + ColumnFamilyOptions cf_univ_opts; + cf_univ_opts.OptimizeUniversalStyleCompaction(); + + ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"}, + {cf_level_opts, cf_univ_opts}, + kOptionsFileName, env_.get())); + + RocksDBOptionsParser parser; + ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); + + { + Options old_default_opts; + old_default_opts.OldDefaults(); + ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base); + ASSERT_EQ(5000, old_default_opts.max_open_files); + ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); + ASSERT_EQ(WALRecoveryMode::kTolerateCorruptedTailRecords, + old_default_opts.wal_recovery_mode); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(4, 6); + ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base); + ASSERT_EQ(5000, old_default_opts.max_open_files); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(4, 7); + ASSERT_NE(10 * 1048576, old_default_opts.max_bytes_for_level_base); + ASSERT_NE(4, old_default_opts.table_cache_numshardbits); + ASSERT_EQ(5000, old_default_opts.max_open_files); + ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); + } + { + ColumnFamilyOptions old_default_cf_opts; + old_default_cf_opts.OldDefaults(); + ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(4 << 20, old_default_cf_opts.write_buffer_size); + ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(0, old_default_cf_opts.soft_pending_compaction_bytes_limit); + ASSERT_EQ(0, old_default_cf_opts.hard_pending_compaction_bytes_limit); + ASSERT_EQ(CompactionPri::kByCompensatedSize, + old_default_cf_opts.compaction_pri); + } + { + ColumnFamilyOptions old_default_cf_opts; + old_default_cf_opts.OldDefaults(4, 6); + ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(CompactionPri::kByCompensatedSize, + old_default_cf_opts.compaction_pri); + } + { + ColumnFamilyOptions old_default_cf_opts; + old_default_cf_opts.OldDefaults(4, 7); + ASSERT_NE(2 * 1048576, old_default_cf_opts.target_file_size_base); + ASSERT_EQ(CompactionPri::kByCompensatedSize, + old_default_cf_opts.compaction_pri); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(5, 1); + ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(5, 2); + ASSERT_EQ(16 * 1024U * 1024U, old_default_opts.delayed_write_rate); + ASSERT_TRUE(old_default_opts.compaction_pri == + CompactionPri::kByCompensatedSize); + } + { + Options old_default_opts; + old_default_opts.OldDefaults(5, 18); + ASSERT_TRUE(old_default_opts.compaction_pri == + CompactionPri::kByCompensatedSize); + } + + Options small_opts; + small_opts.OptimizeForSmallDb(); + ASSERT_EQ(2 << 20, small_opts.write_buffer_size); + ASSERT_EQ(5000, small_opts.max_open_files); +} + +class OptionsSanityCheckTest : public OptionsParserTest { + public: + OptionsSanityCheckTest() {} + + protected: + Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts, + OptionsSanityCheckLevel level) { + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, env_.get(), + level); + } + + Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) { + Status s = env_->DeleteFile(kOptionsFileName); + if (!s.ok()) { + return s; + } + return PersistRocksDBOptions(DBOptions(), {"default"}, {cf_opts}, + kOptionsFileName, env_.get()); + } + + const std::string kOptionsFileName = "OPTIONS"; +}; + +TEST_F(OptionsSanityCheckTest, SanityCheck) { + ColumnFamilyOptions opts; + Random rnd(301); + + // default ColumnFamilyOptions + { + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + + // prefix_extractor + { + // Okay to change prefix_extractor form nullptr to non-nullptr + ASSERT_EQ(opts.prefix_extractor.get(), nullptr); + opts.prefix_extractor.reset(NewCappedPrefixTransform(10)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + + // use same prefix extractor but with different parameter + opts.prefix_extractor.reset(NewCappedPrefixTransform(15)); + // expect pass only in kSanityLevelLooselyCompatible + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // repeat the test with FixedPrefixTransform + opts.prefix_extractor.reset(NewFixedPrefixTransform(10)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change of prefix_extractor + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + + // use same prefix extractor but with different parameter + opts.prefix_extractor.reset(NewFixedPrefixTransform(15)); + // expect pass only in kSanityLevelLooselyCompatible + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // Change prefix extractor from non-nullptr to nullptr + opts.prefix_extractor.reset(); + // expect pass as it's safe to change prefix_extractor + // from non-null to null + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + } + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + + // table_factory + { + for (int tb = 0; tb <= 2; ++tb) { + // change the table factory + opts.table_factory.reset(test::RandomTableFactory(&rnd, tb)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + } + + // merge_operator + { + // Test when going from nullptr -> merge operator + opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + + for (int test = 0; test < 5; ++test) { + // change the merge operator + opts.merge_operator.reset(test::RandomMergeOperator(&rnd)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + + // Test when going from merge operator -> nullptr + opts.merge_operator = nullptr; + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + + // compaction_filter + { + for (int test = 0; test < 5; ++test) { + // change the compaction filter + opts.compaction_filter = test::RandomCompactionFilter(&rnd); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + delete opts.compaction_filter; + opts.compaction_filter = nullptr; + } + } + + // compaction_filter_factory + { + for (int test = 0; test < 5; ++test) { + // change the compaction filter factory + opts.compaction_filter_factory.reset( + test::RandomCompactionFilterFactory(&rnd)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + } +} + +namespace { +bool IsEscapedString(const std::string& str) { + for (size_t i = 0; i < str.size(); ++i) { + if (str[i] == '\\') { + // since we already handle those two consecutive '\'s in + // the next if-then branch, any '\' appear at the end + // of an escaped string in such case is not valid. + if (i == str.size() - 1) { + return false; + } + if (str[i + 1] == '\\') { + // if there're two consecutive '\'s, skip the second one. + i++; + continue; + } + switch (str[i + 1]) { + case ':': + case '\\': + case '#': + continue; + default: + // if true, '\' together with str[i + 1] is not a valid escape. + if (UnescapeChar(str[i + 1]) == str[i + 1]) { + return false; + } + } + } else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) { + return false; + } + } + return true; +} +} // namespace + +TEST_F(OptionsParserTest, IntegerParsing) { + ASSERT_EQ(ParseUint64("18446744073709551615"), 18446744073709551615U); + ASSERT_EQ(ParseUint32("4294967295"), 4294967295U); + ASSERT_EQ(ParseSizeT("18446744073709551615"), 18446744073709551615U); + ASSERT_EQ(ParseInt64("9223372036854775807"), 9223372036854775807U); + ASSERT_EQ(ParseInt64("-9223372036854775808"), port::kMinInt64); + ASSERT_EQ(ParseInt32("2147483647"), 2147483647U); + ASSERT_EQ(ParseInt32("-2147483648"), port::kMinInt32); + ASSERT_EQ(ParseInt("-32767"), -32767); + ASSERT_EQ(ParseDouble("-1.234567"), -1.234567); +} + +TEST_F(OptionsParserTest, EscapeOptionString) { + ASSERT_EQ(UnescapeOptionString( + "This is a test string with \\# \\: and \\\\ escape chars."), + "This is a test string with # : and \\ escape chars."); + + ASSERT_EQ( + EscapeOptionString("This is a test string with # : and \\ escape chars."), + "This is a test string with \\# \\: and \\\\ escape chars."); + + std::string readible_chars = + "A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU" + "YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>" + "<MNBVCXZ\\\" should be okay to \\#\\\\\\:\\#\\#\\#\\ " + "be serialized and deserialized"; + + std::string escaped_string = EscapeOptionString(readible_chars); + ASSERT_TRUE(IsEscapedString(escaped_string)); + // This two transformations should be canceled and should output + // the original input. + ASSERT_EQ(UnescapeOptionString(escaped_string), readible_chars); + + std::string all_chars; + for (unsigned char c = 0;; ++c) { + all_chars += c; + if (c == 255) { + break; + } + } + escaped_string = EscapeOptionString(all_chars); + ASSERT_TRUE(IsEscapedString(escaped_string)); + ASSERT_EQ(UnescapeOptionString(escaped_string), all_chars); + + ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment( + " A simple statement with a comment. # like this :)"), + "A simple statement with a comment."); + + ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment( + "Escape \\# and # comment together ."), + "Escape \\# and"); +} +#endif // !ROCKSDB_LITE +} // namespace rocksdb + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} |