diff options
Diffstat (limited to 'src/rocksdb/options/options_parser.cc')
-rw-r--r-- | src/rocksdb/options/options_parser.cc | 727 |
1 files changed, 727 insertions, 0 deletions
diff --git a/src/rocksdb/options/options_parser.cc b/src/rocksdb/options/options_parser.cc new file mode 100644 index 000000000..562a7b214 --- /dev/null +++ b/src/rocksdb/options/options_parser.cc @@ -0,0 +1,727 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#ifndef ROCKSDB_LITE + +#include "options/options_parser.h" + +#include <cmath> +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "file/line_file_reader.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" +#include "options/db_options.h" +#include "options/options_helper.h" +#include "port/port.h" +#include "rocksdb/convenience.h" +#include "rocksdb/db.h" +#include "rocksdb/utilities/options_type.h" +#include "test_util/sync_point.h" +#include "util/cast_util.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { + +static const std::string option_file_header = + "# This is a RocksDB option file.\n" + "#\n" + "# For detailed file format spec, please refer to the example file\n" + "# in examples/rocksdb_option_file_example.ini\n" + "#\n" + "\n"; + +Status PersistRocksDBOptions(const DBOptions& db_opt, + const std::vector<std::string>& cf_names, + const std::vector<ColumnFamilyOptions>& cf_opts, + const std::string& file_name, FileSystem* fs) { + ConfigOptions + config_options; // Use default for escaped(true) and check (exact) + config_options.delimiter = "\n "; + // Do not invoke PrepareOptions when we are doing validation. + config_options.invoke_prepare_options = false; + // If a readahead size was set in the input options, use it + if (db_opt.log_readahead_size > 0) { + config_options.file_readahead_size = db_opt.log_readahead_size; + } + return PersistRocksDBOptions(config_options, db_opt, cf_names, cf_opts, + file_name, fs); +} + +Status PersistRocksDBOptions(const ConfigOptions& config_options_in, + const DBOptions& db_opt, + const std::vector<std::string>& cf_names, + const std::vector<ColumnFamilyOptions>& cf_opts, + const std::string& file_name, FileSystem* fs) { + ConfigOptions config_options = config_options_in; + config_options.delimiter = "\n "; // Override the default to nl + + TEST_SYNC_POINT("PersistRocksDBOptions:start"); + if (cf_names.size() != cf_opts.size()) { + return Status::InvalidArgument( + "cf_names.size() and cf_opts.size() must be the same"); + } + std::unique_ptr<FSWritableFile> wf; + + Status s = + fs->NewWritableFile(file_name, FileOptions(), &wf, nullptr); + if (!s.ok()) { + return s; + } + std::unique_ptr<WritableFileWriter> writable; + writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(), + nullptr /* statistics */)); + + std::string options_file_content; + + s = writable->Append( + option_file_header + "[" + opt_section_titles[kOptionSectionVersion] + + "]\n" + " rocksdb_version=" + + std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR) + + "." + std::to_string(ROCKSDB_PATCH) + "\n"); + if (s.ok()) { + s = writable->Append( + " options_file_version=" + std::to_string(ROCKSDB_OPTION_FILE_MAJOR) + + "." + std::to_string(ROCKSDB_OPTION_FILE_MINOR) + "\n"); + } + if (s.ok()) { + s = writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] + + "]\n "); + } + + if (s.ok()) { + s = GetStringFromDBOptions(config_options, db_opt, &options_file_content); + } + if (s.ok()) { + s = writable->Append(options_file_content + "\n"); + } + + for (size_t i = 0; s.ok() && i < cf_opts.size(); ++i) { + // CFOptions section + s = writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] + + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); + if (s.ok()) { + s = GetStringFromColumnFamilyOptions(config_options, cf_opts[i], + &options_file_content); + } + if (s.ok()) { + s = writable->Append(options_file_content + "\n"); + } + // TableOptions section + auto* tf = cf_opts[i].table_factory.get(); + if (tf != nullptr) { + if (s.ok()) { + s = writable->Append( + "[" + opt_section_titles[kOptionSectionTableOptions] + tf->Name() + + " \"" + EscapeOptionString(cf_names[i]) + "\"]\n "); + } + if (s.ok()) { + options_file_content.clear(); + s = tf->GetOptionString(config_options, &options_file_content); + } + if (s.ok()) { + s = writable->Append(options_file_content + "\n"); + } + } + } + if (s.ok()) { + s = writable->Sync(true /* use_fsync */); + } + if (s.ok()) { + s = writable->Close(); + } + if (s.ok()) { + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + config_options, db_opt, cf_names, cf_opts, file_name, fs); + } + return s; +} + +RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); } + +void RocksDBOptionsParser::Reset() { + db_opt_ = DBOptions(); + db_opt_map_.clear(); + cf_names_.clear(); + cf_opts_.clear(); + cf_opt_maps_.clear(); + has_version_section_ = false; + has_db_options_ = false; + has_default_cf_options_ = false; + for (int i = 0; i < 3; ++i) { + db_version[i] = 0; + opt_file_version[i] = 0; + } +} + +bool RocksDBOptionsParser::IsSection(const std::string& line) { + if (line.size() < 2) { + return false; + } + if (line[0] != '[' || line[line.size() - 1] != ']') { + return false; + } + return true; +} + +Status RocksDBOptionsParser::ParseSection(OptionSection* section, + std::string* title, + std::string* argument, + const std::string& line, + const int line_num) { + *section = kOptionSectionUnknown; + // A section is of the form [<SectionName> "<SectionArg>"], where + // "<SectionArg>" is optional. + size_t arg_start_pos = line.find("\""); + size_t arg_end_pos = line.rfind("\""); + // The following if-then check tries to identify whether the input + // section has the optional section argument. + if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) { + *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true); + *argument = UnescapeOptionString( + line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1)); + } else { + *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true); + *argument = ""; + } + for (int i = 0; i < kOptionSectionUnknown; ++i) { + if (title->find(opt_section_titles[i]) == 0) { + if (i == kOptionSectionVersion || i == kOptionSectionDBOptions || + i == kOptionSectionCFOptions) { + if (title->size() == opt_section_titles[i].size()) { + // if true, then it indicats equal + *section = static_cast<OptionSection>(i); + return CheckSection(*section, *argument, line_num); + } + } else if (i == kOptionSectionTableOptions) { + // This type of sections has a sufffix at the end of the + // section title + if (title->size() > opt_section_titles[i].size()) { + *section = static_cast<OptionSection>(i); + return CheckSection(*section, *argument, line_num); + } + } + } + } + return Status::InvalidArgument(std::string("Unknown section ") + line); +} + +Status RocksDBOptionsParser::InvalidArgument(const int line_num, + const std::string& message) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error] ", + message + " (at line " + std::to_string(line_num) + ")"); +} + +Status RocksDBOptionsParser::ParseStatement(std::string* name, + std::string* value, + const std::string& line, + const int line_num) { + size_t eq_pos = line.find("="); + if (eq_pos == std::string::npos) { + return InvalidArgument(line_num, "A valid statement must have a '='."); + } + + *name = TrimAndRemoveComment(line.substr(0, eq_pos), true); + *value = + TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1)); + if (name->empty()) { + return InvalidArgument(line_num, + "A valid statement must have a variable name."); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs, + bool ignore_unknown_options, + size_t file_readahead_size) { + ConfigOptions + config_options; // Use default for escaped(true) and check (exact) + config_options.ignore_unknown_options = ignore_unknown_options; + if (file_readahead_size > 0) { + config_options.file_readahead_size = file_readahead_size; + } + return Parse(config_options, file_name, fs); +} + +Status RocksDBOptionsParser::Parse(const ConfigOptions& config_options_in, + const std::string& file_name, + FileSystem* fs) { + Reset(); + ConfigOptions config_options = config_options_in; + + std::unique_ptr<FSSequentialFile> seq_file; + Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file, + nullptr); + if (!s.ok()) { + return s; + } + LineFileReader lf_reader(std::move(seq_file), file_name, + config_options.file_readahead_size); + + OptionSection section = kOptionSectionUnknown; + std::string title; + std::string argument; + std::unordered_map<std::string, std::string> opt_map; + std::string line; + // we only support single-lined statement. + while (lf_reader.ReadLine(&line, Env::IO_TOTAL /* rate_limiter_priority */)) { + int line_num = static_cast<int>(lf_reader.GetLineNumber()); + line = TrimAndRemoveComment(line); + if (line.empty()) { + continue; + } + if (IsSection(line)) { + s = EndSection(config_options, section, title, argument, opt_map); + opt_map.clear(); + if (!s.ok()) { + return s; + } + + // If the option file is not generated by a higher minor version, + // there shouldn't be any unknown option. + if (config_options.ignore_unknown_options && + section == kOptionSectionVersion) { + if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR && + db_version[1] <= ROCKSDB_MINOR)) { + config_options.ignore_unknown_options = false; + } + } + + s = ParseSection(§ion, &title, &argument, line, line_num); + if (!s.ok()) { + return s; + } + } else { + std::string name; + std::string value; + s = ParseStatement(&name, &value, line, line_num); + if (!s.ok()) { + return s; + } + opt_map.insert({name, value}); + } + } + s = lf_reader.GetStatus(); + if (!s.ok()) { + return s; + } + + s = EndSection(config_options, section, title, argument, opt_map); + opt_map.clear(); + if (!s.ok()) { + return s; + } + return ValidityCheck(); +} + +Status RocksDBOptionsParser::CheckSection(const OptionSection section, + const std::string& section_arg, + const int line_num) { + if (section == kOptionSectionDBOptions) { + if (has_db_options_) { + return InvalidArgument( + line_num, + "More than one DBOption section found in the option config file"); + } + has_db_options_ = true; + } else if (section == kOptionSectionCFOptions) { + bool is_default_cf = (section_arg == kDefaultColumnFamilyName); + if (cf_opts_.size() == 0 && !is_default_cf) { + return InvalidArgument( + line_num, + "Default column family must be the first CFOptions section " + "in the option config file"); + } else if (cf_opts_.size() != 0 && is_default_cf) { + return InvalidArgument( + line_num, + "Default column family must be the first CFOptions section " + "in the optio/n config file"); + } else if (GetCFOptions(section_arg) != nullptr) { + return InvalidArgument( + line_num, + "Two identical column families found in option config file"); + } + has_default_cf_options_ |= is_default_cf; + } else if (section == kOptionSectionTableOptions) { + if (GetCFOptions(section_arg) == nullptr) { + return InvalidArgument( + line_num, std::string( + "Does not find a matched column family name in " + "TableOptions section. Column Family Name:") + + section_arg); + } + } else if (section == kOptionSectionVersion) { + if (has_version_section_) { + return InvalidArgument( + line_num, + "More than one Version section found in the option config file."); + } + has_version_section_ = true; + } + return Status::OK(); +} + +Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name, + const std::string& ver_string, + const int max_count, + int* version) { + int version_index = 0; + int current_number = 0; + int current_digit_count = 0; + bool has_dot = false; + for (int i = 0; i < max_count; ++i) { + version[i] = 0; + } + constexpr int kBufferSize = 200; + char buffer[kBufferSize]; + for (size_t i = 0; i < ver_string.size(); ++i) { + if (ver_string[i] == '.') { + if (version_index >= max_count - 1) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s can only contains at most %d dots.", + ver_name.c_str(), max_count - 1); + return Status::InvalidArgument(buffer); + } + if (current_digit_count == 0) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s must have at least one digit before each dot.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + version[version_index++] = current_number; + current_number = 0; + current_digit_count = 0; + has_dot = true; + } else if (isdigit(ver_string[i])) { + current_number = current_number * 10 + (ver_string[i] - '0'); + current_digit_count++; + } else { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s can only contains dots and numbers.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + } + version[version_index] = current_number; + if (has_dot && current_digit_count == 0) { + snprintf(buffer, sizeof(buffer) - 1, + "A valid %s must have at least one digit after each dot.", + ver_name.c_str()); + return Status::InvalidArgument(buffer); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::EndSection( + const ConfigOptions& config_options, const OptionSection section, + const std::string& section_title, const std::string& section_arg, + const std::unordered_map<std::string, std::string>& opt_map) { + Status s; + if (section == kOptionSectionDBOptions) { + s = GetDBOptionsFromMap(config_options, DBOptions(), opt_map, &db_opt_); + if (!s.ok()) { + return s; + } + db_opt_map_ = opt_map; + } else if (section == kOptionSectionCFOptions) { + // This condition should be ensured earlier in ParseSection + // so we make an assertion here. + assert(GetCFOptions(section_arg) == nullptr); + cf_names_.emplace_back(section_arg); + cf_opts_.emplace_back(); + s = GetColumnFamilyOptionsFromMap(config_options, ColumnFamilyOptions(), + opt_map, &cf_opts_.back()); + if (!s.ok()) { + return s; + } + // keep the parsed string. + cf_opt_maps_.emplace_back(opt_map); + } else if (section == kOptionSectionTableOptions) { + assert(GetCFOptions(section_arg) != nullptr); + auto* cf_opt = GetCFOptionsImpl(section_arg); + if (cf_opt == nullptr) { + return Status::InvalidArgument( + "The specified column family must be defined before the " + "TableOptions section:", + section_arg); + } + // Ignore error as table factory deserialization is optional + cf_opt->table_factory.reset(); + s = TableFactory::CreateFromString( + config_options, + section_title.substr( + opt_section_titles[kOptionSectionTableOptions].size()), + &(cf_opt->table_factory)); + if (s.ok() && cf_opt->table_factory != nullptr) { + s = cf_opt->table_factory->ConfigureFromMap(config_options, opt_map); + // Translate any errors (NotFound, NotSupported, to InvalidArgument + if (s.ok() || s.IsInvalidArgument()) { + return s; + } else { + return Status::InvalidArgument(s.getState()); + } + } else { + // Return OK for not supported table factories as TableFactory + // Deserialization is optional. + cf_opt->table_factory.reset(); + return Status::OK(); + } + } else if (section == kOptionSectionVersion) { + for (const auto& pair : opt_map) { + if (pair.first == "rocksdb_version") { + s = ParseVersionNumber(pair.first, pair.second, 3, db_version); + if (!s.ok()) { + return s; + } + } else if (pair.first == "options_file_version") { + s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version); + if (!s.ok()) { + return s; + } + if (opt_file_version[0] < 1) { + return Status::InvalidArgument( + "A valid options_file_version must be at least 1."); + } + } + } + } + return s; +} + +Status RocksDBOptionsParser::ValidityCheck() { + if (!has_db_options_) { + return Status::Corruption( + "A RocksDB Option file must have a single DBOptions section"); + } + if (!has_default_cf_options_) { + return Status::Corruption( + "A RocksDB Option file must have a single CFOptions:default section"); + } + + return Status::OK(); +} + +std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line, + bool trim_only) { + size_t start = 0; + size_t end = line.size(); + + // we only support "#" style comment + if (!trim_only) { + size_t search_pos = 0; + while (search_pos < line.size()) { + size_t comment_pos = line.find('#', search_pos); + if (comment_pos == std::string::npos) { + break; + } + if (comment_pos == 0 || line[comment_pos - 1] != '\\') { + end = comment_pos; + break; + } + search_pos = comment_pos + 1; + } + } + + while (start < end && isspace(line[start]) != 0) { + ++start; + } + + // start < end implies end > 0. + while (start < end && isspace(line[end - 1]) != 0) { + --end; + } + + if (start < end) { + return line.substr(start, end - start); + } + + return ""; +} + +Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + const ConfigOptions& config_options_in, const DBOptions& db_opt, + const std::vector<std::string>& cf_names, + const std::vector<ColumnFamilyOptions>& cf_opts, + const std::string& file_name, FileSystem* fs) { + RocksDBOptionsParser parser; + ConfigOptions config_options = config_options_in; + config_options.invoke_prepare_options = + false; // No need to do a prepare for verify + if (config_options.sanity_level < ConfigOptions::kSanityLevelExactMatch) { + // If we are not doing an exact comparison, we should ignore + // unsupported options, as they may cause the Parse to fail + // (if the ObjectRegistry is not initialized) + config_options.ignore_unsupported_options = true; + } + Status s = parser.Parse(config_options, file_name, fs); + if (!s.ok()) { + return s; + } + + // Verify DBOptions + s = VerifyDBOptions(config_options, db_opt, *parser.db_opt(), + parser.db_opt_map()); + if (!s.ok()) { + return s; + } + + // Verify ColumnFamily Name + if (cf_names.size() != parser.cf_names()->size()) { + if (config_options.sanity_level >= + ConfigOptions::kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionParser Error] The persisted options does not have " + "the same number of column family names as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column family " + "names than that of the specified one."); + } + } + for (size_t i = 0; i < cf_names.size(); ++i) { + if (cf_names[i] != parser.cf_names()->at(i)) { + return Status::InvalidArgument( + "[RocksDBOptionParser Error] The persisted options and the db" + "instance does not have the same name for column family ", + std::to_string(i)); + } + } + + // Verify Column Family Options + if (cf_opts.size() != parser.cf_opts()->size()) { + if (config_options.sanity_level >= + ConfigOptions::kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options does not have the same number of " + "column families as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column families " + "than that of the specified number."); + } + } + for (size_t i = 0; i < cf_opts.size(); ++i) { + s = VerifyCFOptions(config_options, cf_opts[i], parser.cf_opts()->at(i), + &(parser.cf_opt_maps()->at(i))); + if (!s.ok()) { + return s; + } + s = VerifyTableFactory(config_options, cf_opts[i].table_factory.get(), + parser.cf_opts()->at(i).table_factory.get()); + if (!s.ok()) { + return s; + } + } + + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyDBOptions( + const ConfigOptions& config_options, const DBOptions& base_opt, + const DBOptions& file_opt, + const std::unordered_map<std::string, std::string>* opt_map) { + auto base_config = DBOptionsAsConfigurable(base_opt, opt_map); + auto file_config = DBOptionsAsConfigurable(file_opt, opt_map); + std::string mismatch; + if (!base_config->AreEquivalent(config_options, file_config.get(), + &mismatch)) { + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + std::string base_value; + std::string file_value; + int offset = snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on DBOptions::%s -- ", + mismatch.c_str()); + Status s = base_config->GetOption(config_options, mismatch, &base_value); + if (s.ok()) { + s = file_config->GetOption(config_options, mismatch, &file_value); + } + assert(offset >= 0); + assert(static_cast<size_t>(offset) < sizeof(buffer)); + if (s.ok()) { + snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset), + "-- The specified one is %s while the persisted one is %s.\n", + base_value.c_str(), file_value.c_str()); + } else { + snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset), + "-- Unable to re-serialize an option: %s.\n", + s.ToString().c_str()); + } + return Status::InvalidArgument(Slice(buffer, strlen(buffer))); + } + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyCFOptions( + const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, + const ColumnFamilyOptions& file_opt, + const std::unordered_map<std::string, std::string>* opt_map) { + auto base_config = CFOptionsAsConfigurable(base_opt, opt_map); + auto file_config = CFOptionsAsConfigurable(file_opt, opt_map); + std::string mismatch; + if (!base_config->AreEquivalent(config_options, file_config.get(), + &mismatch)) { + std::string base_value; + std::string file_value; + // The options do not match + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + Status s = base_config->GetOption(config_options, mismatch, &base_value); + if (s.ok()) { + s = file_config->GetOption(config_options, mismatch, &file_value); + } + int offset = snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on ColumnFamilyOptions::%s", + mismatch.c_str()); + assert(offset >= 0); + assert(static_cast<size_t>(offset) < sizeof(buffer)); + if (s.ok()) { + snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset), + "--- The specified one is %s while the persisted one is %s.\n", + base_value.c_str(), file_value.c_str()); + } else { + snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset), + "--- Unable to re-serialize an option: %s.\n", + s.ToString().c_str()); + } + return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); + } // For each option + return Status::OK(); +} + +Status RocksDBOptionsParser::VerifyTableFactory( + const ConfigOptions& config_options, const TableFactory* base_tf, + const TableFactory* file_tf) { + std::string mismatch; + if (base_tf && file_tf) { + if (config_options.sanity_level > ConfigOptions::kSanityLevelNone && + std::string(base_tf->Name()) != std::string(file_tf->Name())) { + return Status::Corruption( + "[RocksDBOptionsParser]: " + "failed the verification on TableFactory->Name()"); + } else if (!base_tf->AreEquivalent(config_options, file_tf, &mismatch)) { + return Status::Corruption(std::string("[RocksDBOptionsParser]:" + "failed the verification on ") + + base_tf->Name() + "::", + mismatch); + } + } else { + // TODO(yhchiang): further support sanity check here + } + return Status::OK(); +} +} // namespace ROCKSDB_NAMESPACE + +#endif // !ROCKSDB_LITE |