diff options
Diffstat (limited to '')
| -rw-r--r-- | src/rocksdb/tools/ldb_cmd.cc | 3437 |
1 files changed, 3437 insertions, 0 deletions
diff --git a/src/rocksdb/tools/ldb_cmd.cc b/src/rocksdb/tools/ldb_cmd.cc new file mode 100644 index 000000000..48d32d2ef --- /dev/null +++ b/src/rocksdb/tools/ldb_cmd.cc @@ -0,0 +1,3437 @@ + +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +#ifndef ROCKSDB_LITE +#include "rocksdb/utilities/ldb_cmd.h" + +#include <cinttypes> + +#include "db/db_impl/db_impl.h" +#include "db/dbformat.h" +#include "db/log_reader.h" +#include "db/write_batch_internal.h" +#include "env/composite_env_wrapper.h" +#include "file/filename.h" +#include "port/port_dirent.h" +#include "rocksdb/cache.h" +#include "rocksdb/file_checksum.h" +#include "rocksdb/table_properties.h" +#include "rocksdb/utilities/backupable_db.h" +#include "rocksdb/utilities/checkpoint.h" +#include "rocksdb/utilities/debug.h" +#include "rocksdb/utilities/options_util.h" +#include "rocksdb/write_batch.h" +#include "rocksdb/write_buffer_manager.h" +#include "table/scoped_arena_iterator.h" +#include "tools/ldb_cmd_impl.h" +#include "tools/sst_dump_tool_imp.h" +#include "util/cast_util.h" +#include "util/coding.h" +#include "util/file_checksum_helper.h" +#include "util/stderr_logger.h" +#include "util/string_util.h" +#include "utilities/merge_operators.h" +#include "utilities/ttl/db_ttl_impl.h" + +#include <cstdlib> +#include <ctime> +#include <fstream> +#include <functional> +#include <iostream> +#include <limits> +#include <sstream> +#include <stdexcept> +#include <string> + +namespace ROCKSDB_NAMESPACE { + +class FileChecksumFuncCrc32c; + +const std::string LDBCommand::ARG_ENV_URI = "env_uri"; +const std::string LDBCommand::ARG_DB = "db"; +const std::string LDBCommand::ARG_PATH = "path"; +const std::string LDBCommand::ARG_SECONDARY_PATH = "secondary_path"; +const std::string LDBCommand::ARG_HEX = "hex"; +const std::string LDBCommand::ARG_KEY_HEX = "key_hex"; +const std::string LDBCommand::ARG_VALUE_HEX = "value_hex"; +const std::string LDBCommand::ARG_CF_NAME = "column_family"; +const std::string LDBCommand::ARG_TTL = "ttl"; +const std::string LDBCommand::ARG_TTL_START = "start_time"; +const std::string LDBCommand::ARG_TTL_END = "end_time"; +const std::string LDBCommand::ARG_TIMESTAMP = "timestamp"; +const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options"; +const std::string LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS = + "ignore_unknown_options"; +const std::string LDBCommand::ARG_FROM = "from"; +const std::string LDBCommand::ARG_TO = "to"; +const std::string LDBCommand::ARG_MAX_KEYS = "max_keys"; +const std::string LDBCommand::ARG_BLOOM_BITS = "bloom_bits"; +const std::string LDBCommand::ARG_FIX_PREFIX_LEN = "fix_prefix_len"; +const std::string LDBCommand::ARG_COMPRESSION_TYPE = "compression_type"; +const std::string LDBCommand::ARG_COMPRESSION_MAX_DICT_BYTES = + "compression_max_dict_bytes"; +const std::string LDBCommand::ARG_BLOCK_SIZE = "block_size"; +const std::string LDBCommand::ARG_AUTO_COMPACTION = "auto_compaction"; +const std::string LDBCommand::ARG_DB_WRITE_BUFFER_SIZE = "db_write_buffer_size"; +const std::string LDBCommand::ARG_WRITE_BUFFER_SIZE = "write_buffer_size"; +const std::string LDBCommand::ARG_FILE_SIZE = "file_size"; +const std::string LDBCommand::ARG_CREATE_IF_MISSING = "create_if_missing"; +const std::string LDBCommand::ARG_NO_VALUE = "no_value"; + +const char* LDBCommand::DELIM = " ==> "; + +namespace { + +void DumpWalFile(Options options, std::string wal_file, bool print_header, + bool print_values, bool is_write_committed, + LDBCommandExecuteResult* exec_state); + +void DumpSstFile(Options options, std::string filename, bool output_hex, + bool show_properties); +}; + +LDBCommand* LDBCommand::InitFromCmdLineArgs( + int argc, char** argv, const Options& options, + const LDBOptions& ldb_options, + const std::vector<ColumnFamilyDescriptor>* column_families) { + std::vector<std::string> args; + for (int i = 1; i < argc; i++) { + args.push_back(argv[i]); + } + return InitFromCmdLineArgs(args, options, ldb_options, column_families, + SelectCommand); +} + +/** + * Parse the command-line arguments and create the appropriate LDBCommand2 + * instance. + * The command line arguments must be in the following format: + * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] .. + * COMMAND <PARAM1> <PARAM2> ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] .. + * This is similar to the command line format used by HBaseClientTool. + * Command name is not included in args. + * Returns nullptr if the command-line cannot be parsed. + */ +LDBCommand* LDBCommand::InitFromCmdLineArgs( + const std::vector<std::string>& args, const Options& options, + const LDBOptions& ldb_options, + const std::vector<ColumnFamilyDescriptor>* /*column_families*/, + const std::function<LDBCommand*(const ParsedParams&)>& selector) { + // --x=y command line arguments are added as x->y map entries in + // parsed_params.option_map. + // + // Command-line arguments of the form --hex end up in this array as hex to + // parsed_params.flags + ParsedParams parsed_params; + + // Everything other than option_map and flags. Represents commands + // and their parameters. For eg: put key1 value1 go into this vector. + std::vector<std::string> cmdTokens; + + const std::string OPTION_PREFIX = "--"; + + for (const auto& arg : args) { + if (arg[0] == '-' && arg[1] == '-'){ + std::vector<std::string> splits = StringSplit(arg, '='); + // --option_name=option_value + if (splits.size() == 2) { + std::string optionKey = splits[0].substr(OPTION_PREFIX.size()); + parsed_params.option_map[optionKey] = splits[1]; + } else if (splits.size() == 1) { + // --flag_name + std::string optionKey = splits[0].substr(OPTION_PREFIX.size()); + parsed_params.flags.push_back(optionKey); + } else { + // --option_name=option_value, option_value contains '=' + std::string optionKey = splits[0].substr(OPTION_PREFIX.size()); + parsed_params.option_map[optionKey] = + arg.substr(splits[0].length() + 1); + } + } else { + cmdTokens.push_back(arg); + } + } + + if (cmdTokens.size() < 1) { + fprintf(stderr, "Command not specified!"); + return nullptr; + } + + parsed_params.cmd = cmdTokens[0]; + parsed_params.cmd_params.assign(cmdTokens.begin() + 1, cmdTokens.end()); + + LDBCommand* command = selector(parsed_params); + + if (command) { + command->SetDBOptions(options); + command->SetLDBOptions(ldb_options); + } + return command; +} + +LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) { + if (parsed_params.cmd == GetCommand::Name()) { + return new GetCommand(parsed_params.cmd_params, parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == PutCommand::Name()) { + return new PutCommand(parsed_params.cmd_params, parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == BatchPutCommand::Name()) { + return new BatchPutCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == ScanCommand::Name()) { + return new ScanCommand(parsed_params.cmd_params, parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == DeleteCommand::Name()) { + return new DeleteCommand(parsed_params.cmd_params, parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == DeleteRangeCommand::Name()) { + return new DeleteRangeCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == ApproxSizeCommand::Name()) { + return new ApproxSizeCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == DBQuerierCommand::Name()) { + return new DBQuerierCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == CompactorCommand::Name()) { + return new CompactorCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == WALDumperCommand::Name()) { + return new WALDumperCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == ReduceDBLevelsCommand::Name()) { + return new ReduceDBLevelsCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == ChangeCompactionStyleCommand::Name()) { + return new ChangeCompactionStyleCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == DBDumperCommand::Name()) { + return new DBDumperCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == DBLoaderCommand::Name()) { + return new DBLoaderCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == ManifestDumpCommand::Name()) { + return new ManifestDumpCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == FileChecksumDumpCommand::Name()) { + return new FileChecksumDumpCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == ListColumnFamiliesCommand::Name()) { + return new ListColumnFamiliesCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == CreateColumnFamilyCommand::Name()) { + return new CreateColumnFamilyCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == DropColumnFamilyCommand::Name()) { + return new DropColumnFamilyCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == DBFileDumperCommand::Name()) { + return new DBFileDumperCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == InternalDumpCommand::Name()) { + return new InternalDumpCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == CheckConsistencyCommand::Name()) { + return new CheckConsistencyCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == CheckPointCommand::Name()) { + return new CheckPointCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == RepairCommand::Name()) { + return new RepairCommand(parsed_params.cmd_params, parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == BackupCommand::Name()) { + return new BackupCommand(parsed_params.cmd_params, parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == RestoreCommand::Name()) { + return new RestoreCommand(parsed_params.cmd_params, + parsed_params.option_map, parsed_params.flags); + } else if (parsed_params.cmd == WriteExternalSstFilesCommand::Name()) { + return new WriteExternalSstFilesCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == IngestExternalSstFilesCommand::Name()) { + return new IngestExternalSstFilesCommand(parsed_params.cmd_params, + parsed_params.option_map, + parsed_params.flags); + } else if (parsed_params.cmd == ListFileRangeDeletesCommand::Name()) { + return new ListFileRangeDeletesCommand(parsed_params.option_map, + parsed_params.flags); + } + return nullptr; +} + +/* Run the command, and return the execute result. */ +void LDBCommand::Run() { + if (!exec_state_.IsNotStarted()) { + return; + } + + if (!options_.env || options_.env == Env::Default()) { + Env* env = Env::Default(); + Status s = Env::LoadEnv(env_uri_, &env, &env_guard_); + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "LoadEnv: %s\n", s.ToString().c_str()); + exec_state_ = LDBCommandExecuteResult::Failed(s.ToString()); + return; + } + options_.env = env; + } + + options_.file_system.reset(new LegacyFileSystemWrapper(options_.env)); + + if (db_ == nullptr && !NoDBOpen()) { + OpenDB(); + if (exec_state_.IsFailed() && try_load_options_) { + // We don't always return if there is a failure because a WAL file or + // manifest file can be given to "dump" command so we should continue. + // --try_load_options is not valid in those cases. + return; + } + } + + // We'll intentionally proceed even if the DB can't be opened because users + // can also specify a filename, not just a directory. + DoCommand(); + + if (exec_state_.IsNotStarted()) { + exec_state_ = LDBCommandExecuteResult::Succeed(""); + } + + if (db_ != nullptr) { + CloseDB(); + } +} + +LDBCommand::LDBCommand(const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags, bool is_read_only, + const std::vector<std::string>& valid_cmd_line_options) + : db_(nullptr), + db_ttl_(nullptr), + is_read_only_(is_read_only), + is_key_hex_(false), + is_value_hex_(false), + is_db_ttl_(false), + timestamp_(false), + try_load_options_(false), + ignore_unknown_options_(false), + create_if_missing_(false), + option_map_(options), + flags_(flags), + valid_cmd_line_options_(valid_cmd_line_options) { + std::map<std::string, std::string>::const_iterator itr = options.find(ARG_DB); + if (itr != options.end()) { + db_path_ = itr->second; + } + + itr = options.find(ARG_ENV_URI); + if (itr != options.end()) { + env_uri_ = itr->second; + } + + itr = options.find(ARG_CF_NAME); + if (itr != options.end()) { + column_family_name_ = itr->second; + } else { + column_family_name_ = kDefaultColumnFamilyName; + } + + itr = options.find(ARG_SECONDARY_PATH); + secondary_path_ = ""; + if (itr != options.end()) { + secondary_path_ = itr->second; + } + + is_key_hex_ = IsKeyHex(options, flags); + is_value_hex_ = IsValueHex(options, flags); + is_db_ttl_ = IsFlagPresent(flags, ARG_TTL); + timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP); + try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS); + ignore_unknown_options_ = IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS); +} + +void LDBCommand::OpenDB() { + if (!create_if_missing_ && try_load_options_) { + Status s = LoadLatestOptions(db_path_, options_.env, &options_, + &column_families_, ignore_unknown_options_); + if (!s.ok() && !s.IsNotFound()) { + // Option file exists but load option file error. + std::string msg = s.ToString(); + exec_state_ = LDBCommandExecuteResult::Failed(msg); + db_ = nullptr; + return; + } + if (options_.env->FileExists(options_.wal_dir).IsNotFound()) { + options_.wal_dir = db_path_; + fprintf( + stderr, + "wal_dir loaded from the option file doesn't exist. Ignore it.\n"); + } + + // If merge operator is not set, set a string append operator. There is + // no harm doing it. + for (auto& cf_entry : column_families_) { + if (!cf_entry.options.merge_operator) { + cf_entry.options.merge_operator = + MergeOperators::CreateStringAppendOperator(':'); + } + } + } + options_ = PrepareOptionsForOpenDB(); + if (!exec_state_.IsNotStarted()) { + return; + } + if (column_families_.empty() && !options_.merge_operator) { + // No harm to add a general merge operator if it is not specified. + options_.merge_operator = MergeOperators::CreateStringAppendOperator(':'); + } + // Open the DB. + Status st; + std::vector<ColumnFamilyHandle*> handles_opened; + if (is_db_ttl_) { + // ldb doesn't yet support TTL DB with multiple column families + if (!column_family_name_.empty() || !column_families_.empty()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "ldb doesn't support TTL DB with multiple column families"); + } + if (!secondary_path_.empty()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Open as secondary is not supported for TTL DB yet."); + } + if (is_read_only_) { + st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true); + } else { + st = DBWithTTL::Open(options_, db_path_, &db_ttl_); + } + db_ = db_ttl_; + } else { + if (column_families_.empty()) { + // Try to figure out column family lists + std::vector<std::string> cf_list; + st = DB::ListColumnFamilies(options_, db_path_, &cf_list); + // There is possible the DB doesn't exist yet, for "create if not + // "existing case". The failure is ignored here. We rely on DB::Open() + // to give us the correct error message for problem with opening + // existing DB. + if (st.ok() && cf_list.size() > 1) { + // Ignore single column family DB. + for (auto cf_name : cf_list) { + column_families_.emplace_back(cf_name, options_); + } + } + } + if (is_read_only_ && secondary_path_.empty()) { + if (column_families_.empty()) { + st = DB::OpenForReadOnly(options_, db_path_, &db_); + } else { + st = DB::OpenForReadOnly(options_, db_path_, column_families_, + &handles_opened, &db_); + } + } else { + if (column_families_.empty()) { + if (secondary_path_.empty()) { + st = DB::Open(options_, db_path_, &db_); + } else { + st = DB::OpenAsSecondary(options_, db_path_, secondary_path_, &db_); + } + } else { + if (secondary_path_.empty()) { + st = DB::Open(options_, db_path_, column_families_, &handles_opened, + &db_); + } else { + st = DB::OpenAsSecondary(options_, db_path_, secondary_path_, + column_families_, &handles_opened, &db_); + } + } + } + } + if (!st.ok()) { + std::string msg = st.ToString(); + exec_state_ = LDBCommandExecuteResult::Failed(msg); + } else if (!handles_opened.empty()) { + assert(handles_opened.size() == column_families_.size()); + bool found_cf_name = false; + for (size_t i = 0; i < handles_opened.size(); i++) { + cf_handles_[column_families_[i].name] = handles_opened[i]; + if (column_family_name_ == column_families_[i].name) { + found_cf_name = true; + } + } + if (!found_cf_name) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Non-existing column family " + column_family_name_); + CloseDB(); + } + } else { + // We successfully opened DB in single column family mode. + assert(column_families_.empty()); + if (column_family_name_ != kDefaultColumnFamilyName) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Non-existing column family " + column_family_name_); + CloseDB(); + } + } +} + +void LDBCommand::CloseDB() { + if (db_ != nullptr) { + for (auto& pair : cf_handles_) { + delete pair.second; + } + delete db_; + db_ = nullptr; + } +} + +ColumnFamilyHandle* LDBCommand::GetCfHandle() { + if (!cf_handles_.empty()) { + auto it = cf_handles_.find(column_family_name_); + if (it == cf_handles_.end()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Cannot find column family " + column_family_name_); + } else { + return it->second; + } + } + return db_->DefaultColumnFamily(); +} + +std::vector<std::string> LDBCommand::BuildCmdLineOptions( + std::vector<std::string> options) { + std::vector<std::string> ret = {ARG_ENV_URI, + ARG_DB, + ARG_SECONDARY_PATH, + ARG_BLOOM_BITS, + ARG_BLOCK_SIZE, + ARG_AUTO_COMPACTION, + ARG_COMPRESSION_TYPE, + ARG_COMPRESSION_MAX_DICT_BYTES, + ARG_WRITE_BUFFER_SIZE, + ARG_FILE_SIZE, + ARG_FIX_PREFIX_LEN, + ARG_TRY_LOAD_OPTIONS, + ARG_IGNORE_UNKNOWN_OPTIONS, + ARG_CF_NAME}; + ret.insert(ret.end(), options.begin(), options.end()); + return ret; +} + +/** + * Parses the specific integer option and fills in the value. + * Returns true if the option is found. + * Returns false if the option is not found or if there is an error parsing the + * value. If there is an error, the specified exec_state is also + * updated. + */ +bool LDBCommand::ParseIntOption( + const std::map<std::string, std::string>& /*options*/, + const std::string& option, int& value, + LDBCommandExecuteResult& exec_state) { + std::map<std::string, std::string>::const_iterator itr = + option_map_.find(option); + if (itr != option_map_.end()) { + try { +#if defined(CYGWIN) + value = strtol(itr->second.c_str(), 0, 10); +#else + value = std::stoi(itr->second); +#endif + return true; + } catch (const std::invalid_argument&) { + exec_state = + LDBCommandExecuteResult::Failed(option + " has an invalid value."); + } catch (const std::out_of_range&) { + exec_state = LDBCommandExecuteResult::Failed( + option + " has a value out-of-range."); + } + } + return false; +} + +/** + * Parses the specified option and fills in the value. + * Returns true if the option is found. + * Returns false otherwise. + */ +bool LDBCommand::ParseStringOption( + const std::map<std::string, std::string>& /*options*/, + const std::string& option, std::string* value) { + auto itr = option_map_.find(option); + if (itr != option_map_.end()) { + *value = itr->second; + return true; + } + return false; +} + +Options LDBCommand::PrepareOptionsForOpenDB() { + ColumnFamilyOptions* cf_opts; + auto column_families_iter = + std::find_if(column_families_.begin(), column_families_.end(), + [this](const ColumnFamilyDescriptor& cf_desc) { + return cf_desc.name == column_family_name_; + }); + if (column_families_iter != column_families_.end()) { + cf_opts = &column_families_iter->options; + } else { + cf_opts = static_cast<ColumnFamilyOptions*>(&options_); + } + DBOptions* db_opts = static_cast<DBOptions*>(&options_); + db_opts->create_if_missing = false; + + std::map<std::string, std::string>::const_iterator itr; + + BlockBasedTableOptions table_options; + bool use_table_options = false; + int bits; + if (ParseIntOption(option_map_, ARG_BLOOM_BITS, bits, exec_state_)) { + if (bits > 0) { + use_table_options = true; + table_options.filter_policy.reset(NewBloomFilterPolicy(bits)); + } else { + exec_state_ = + LDBCommandExecuteResult::Failed(ARG_BLOOM_BITS + " must be > 0."); + } + } + + int block_size; + if (ParseIntOption(option_map_, ARG_BLOCK_SIZE, block_size, exec_state_)) { + if (block_size > 0) { + use_table_options = true; + table_options.block_size = block_size; + } else { + exec_state_ = + LDBCommandExecuteResult::Failed(ARG_BLOCK_SIZE + " must be > 0."); + } + } + + if (use_table_options) { + cf_opts->table_factory.reset(NewBlockBasedTableFactory(table_options)); + } + + itr = option_map_.find(ARG_AUTO_COMPACTION); + if (itr != option_map_.end()) { + cf_opts->disable_auto_compactions = !StringToBool(itr->second); + } + + itr = option_map_.find(ARG_COMPRESSION_TYPE); + if (itr != option_map_.end()) { + std::string comp = itr->second; + if (comp == "no") { + cf_opts->compression = kNoCompression; + } else if (comp == "snappy") { + cf_opts->compression = kSnappyCompression; + } else if (comp == "zlib") { + cf_opts->compression = kZlibCompression; + } else if (comp == "bzip2") { + cf_opts->compression = kBZip2Compression; + } else if (comp == "lz4") { + cf_opts->compression = kLZ4Compression; + } else if (comp == "lz4hc") { + cf_opts->compression = kLZ4HCCompression; + } else if (comp == "xpress") { + cf_opts->compression = kXpressCompression; + } else if (comp == "zstd") { + cf_opts->compression = kZSTD; + } else { + // Unknown compression. + exec_state_ = + LDBCommandExecuteResult::Failed("Unknown compression level: " + comp); + } + } + + int compression_max_dict_bytes; + if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES, + compression_max_dict_bytes, exec_state_)) { + if (compression_max_dict_bytes >= 0) { + cf_opts->compression_opts.max_dict_bytes = compression_max_dict_bytes; + } else { + exec_state_ = LDBCommandExecuteResult::Failed( + ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0."); + } + } + + int db_write_buffer_size; + if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE, + db_write_buffer_size, exec_state_)) { + if (db_write_buffer_size >= 0) { + db_opts->db_write_buffer_size = db_write_buffer_size; + } else { + exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE + + " must be >= 0."); + } + } + + int write_buffer_size; + if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size, + exec_state_)) { + if (write_buffer_size > 0) { + cf_opts->write_buffer_size = write_buffer_size; + } else { + exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE + + " must be > 0."); + } + } + + int file_size; + if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) { + if (file_size > 0) { + cf_opts->target_file_size_base = file_size; + } else { + exec_state_ = + LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0."); + } + } + + if (db_opts->db_paths.size() == 0) { + db_opts->db_paths.emplace_back(db_path_, + std::numeric_limits<uint64_t>::max()); + } + + int fix_prefix_len; + if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len, + exec_state_)) { + if (fix_prefix_len > 0) { + cf_opts->prefix_extractor.reset( + NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len))); + } else { + exec_state_ = + LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0."); + } + } + + // TODO(ajkr): this return value doesn't reflect the CF options changed, so + // subcommands that rely on this won't see the effect of CF-related CLI args. + // Such subcommands need to be changed to properly support CFs. + return options_; +} + +bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key, + std::string* value, bool is_key_hex, + bool is_value_hex) { + size_t pos = line.find(DELIM); + if (pos != std::string::npos) { + *key = line.substr(0, pos); + *value = line.substr(pos + strlen(DELIM)); + if (is_key_hex) { + *key = HexToString(*key); + } + if (is_value_hex) { + *value = HexToString(*value); + } + return true; + } else { + return false; + } +} + +/** + * Make sure that ONLY the command-line options and flags expected by this + * command are specified on the command-line. Extraneous options are usually + * the result of user error. + * Returns true if all checks pass. Else returns false, and prints an + * appropriate error msg to stderr. + */ +bool LDBCommand::ValidateCmdLineOptions() { + for (std::map<std::string, std::string>::const_iterator itr = + option_map_.begin(); + itr != option_map_.end(); ++itr) { + if (std::find(valid_cmd_line_options_.begin(), + valid_cmd_line_options_.end(), + itr->first) == valid_cmd_line_options_.end()) { + fprintf(stderr, "Invalid command-line option %s\n", itr->first.c_str()); + return false; + } + } + + for (std::vector<std::string>::const_iterator itr = flags_.begin(); + itr != flags_.end(); ++itr) { + if (std::find(valid_cmd_line_options_.begin(), + valid_cmd_line_options_.end(), + *itr) == valid_cmd_line_options_.end()) { + fprintf(stderr, "Invalid command-line flag %s\n", itr->c_str()); + return false; + } + } + + if (!NoDBOpen() && option_map_.find(ARG_DB) == option_map_.end() && + option_map_.find(ARG_PATH) == option_map_.end()) { + fprintf(stderr, "Either %s or %s must be specified.\n", ARG_DB.c_str(), + ARG_PATH.c_str()); + return false; + } + + return true; +} + +std::string LDBCommand::HexToString(const std::string& str) { + std::string result; + std::string::size_type len = str.length(); + if (len < 2 || str[0] != '0' || str[1] != 'x') { + fprintf(stderr, "Invalid hex input %s. Must start with 0x\n", str.c_str()); + throw "Invalid hex input"; + } + if (!Slice(str.data() + 2, len - 2).DecodeHex(&result)) { + throw "Invalid hex input"; + } + return result; +} + +std::string LDBCommand::StringToHex(const std::string& str) { + std::string result("0x"); + result.append(Slice(str).ToString(true)); + return result; +} + +std::string LDBCommand::PrintKeyValue(const std::string& key, + const std::string& value, bool is_key_hex, + bool is_value_hex) { + std::string result; + result.append(is_key_hex ? StringToHex(key) : key); + result.append(DELIM); + result.append(is_value_hex ? StringToHex(value) : value); + return result; +} + +std::string LDBCommand::PrintKeyValue(const std::string& key, + const std::string& value, bool is_hex) { + return PrintKeyValue(key, value, is_hex, is_hex); +} + +std::string LDBCommand::HelpRangeCmdArgs() { + std::ostringstream str_stream; + str_stream << " "; + str_stream << "[--" << ARG_FROM << "] "; + str_stream << "[--" << ARG_TO << "] "; + return str_stream.str(); +} + +bool LDBCommand::IsKeyHex(const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) { + return (IsFlagPresent(flags, ARG_HEX) || IsFlagPresent(flags, ARG_KEY_HEX) || + ParseBooleanOption(options, ARG_HEX, false) || + ParseBooleanOption(options, ARG_KEY_HEX, false)); +} + +bool LDBCommand::IsValueHex(const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) { + return (IsFlagPresent(flags, ARG_HEX) || + IsFlagPresent(flags, ARG_VALUE_HEX) || + ParseBooleanOption(options, ARG_HEX, false) || + ParseBooleanOption(options, ARG_VALUE_HEX, false)); +} + +bool LDBCommand::ParseBooleanOption( + const std::map<std::string, std::string>& options, + const std::string& option, bool default_val) { + std::map<std::string, std::string>::const_iterator itr = options.find(option); + if (itr != options.end()) { + std::string option_val = itr->second; + return StringToBool(itr->second); + } + return default_val; +} + +bool LDBCommand::StringToBool(std::string val) { + std::transform(val.begin(), val.end(), val.begin(), + [](char ch) -> char { return (char)::tolower(ch); }); + + if (val == "true") { + return true; + } else if (val == "false") { + return false; + } else { + throw "Invalid value for boolean argument"; + } +} + +CompactorCommand::CompactorCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_FROM, ARG_TO, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX, ARG_TTL})), + null_from_(true), + null_to_(true) { + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_FROM); + if (itr != options.end()) { + null_from_ = false; + from_ = itr->second; + } + + itr = options.find(ARG_TO); + if (itr != options.end()) { + null_to_ = false; + to_ = itr->second; + } + + if (is_key_hex_) { + if (!null_from_) { + from_ = HexToString(from_); + } + if (!null_to_) { + to_ = HexToString(to_); + } + } +} + +void CompactorCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(CompactorCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append("\n"); +} + +void CompactorCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + + Slice* begin = nullptr; + Slice* end = nullptr; + if (!null_from_) { + begin = new Slice(from_); + } + if (!null_to_) { + end = new Slice(to_); + } + + CompactRangeOptions cro; + cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized; + + db_->CompactRange(cro, GetCfHandle(), begin, end); + exec_state_ = LDBCommandExecuteResult::Succeed(""); + + delete begin; + delete end; +} + +// --------------------------------------------------------------------------- +const std::string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal"; +const std::string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load"; +const std::string DBLoaderCommand::ARG_COMPACT = "compact"; + +DBLoaderCommand::DBLoaderCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, + ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL, + ARG_BULK_LOAD, ARG_COMPACT})), + disable_wal_(false), + bulk_load_(false), + compact_(false) { + create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING); + disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL); + bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD); + compact_ = IsFlagPresent(flags, ARG_COMPACT); +} + +void DBLoaderCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(DBLoaderCommand::Name()); + ret.append(" [--" + ARG_CREATE_IF_MISSING + "]"); + ret.append(" [--" + ARG_DISABLE_WAL + "]"); + ret.append(" [--" + ARG_BULK_LOAD + "]"); + ret.append(" [--" + ARG_COMPACT + "]"); + ret.append("\n"); +} + +Options DBLoaderCommand::PrepareOptionsForOpenDB() { + Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.create_if_missing = create_if_missing_; + if (bulk_load_) { + opt.PrepareForBulkLoad(); + } + return opt; +} + +void DBLoaderCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + + WriteOptions write_options; + if (disable_wal_) { + write_options.disableWAL = true; + } + + int bad_lines = 0; + std::string line; + // prefer ifstream getline performance vs that from std::cin istream + std::ifstream ifs_stdin("/dev/stdin"); + std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin; + while (getline(*istream_p, line, '\n')) { + std::string key; + std::string value; + if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) { + db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value)); + } else if (0 == line.find("Keys in range:")) { + // ignore this line + } else if (0 == line.find("Created bg thread 0x")) { + // ignore this line + } else { + bad_lines ++; + } + } + + if (bad_lines > 0) { + std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl; + } + if (compact_) { + db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr); + } +} + +// ---------------------------------------------------------------------------- + +namespace { + +void DumpManifestFile(Options options, std::string file, bool verbose, bool hex, + bool json) { + EnvOptions sopt; + std::string dbname("dummy"); + std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10, + options.table_cache_numshardbits)); + // Notice we are using the default options not through SanitizeOptions(), + // if VersionSet::DumpManifest() depends on any option done by + // SanitizeOptions(), we need to initialize it manually. + options.db_paths.emplace_back("dummy", 0); + options.num_levels = 64; + WriteController wc(options.delayed_write_rate); + WriteBufferManager wb(options.db_write_buffer_size); + ImmutableDBOptions immutable_db_options(options); + VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc, + /*block_cache_tracer=*/nullptr); + Status s = versions.DumpManifest(options, file, verbose, hex, json); + if (!s.ok()) { + fprintf(stderr, "Error in processing file %s %s\n", file.c_str(), + s.ToString().c_str()); + } +} + +} // namespace + +const std::string ManifestDumpCommand::ARG_VERBOSE = "verbose"; +const std::string ManifestDumpCommand::ARG_JSON = "json"; +const std::string ManifestDumpCommand::ARG_PATH = "path"; + +void ManifestDumpCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(ManifestDumpCommand::Name()); + ret.append(" [--" + ARG_VERBOSE + "]"); + ret.append(" [--" + ARG_JSON + "]"); + ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]"); + ret.append("\n"); +} + +ManifestDumpCommand::ManifestDumpCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, false, + BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})), + verbose_(false), + json_(false), + path_("") { + verbose_ = IsFlagPresent(flags, ARG_VERBOSE); + json_ = IsFlagPresent(flags, ARG_JSON); + + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_PATH); + if (itr != options.end()) { + path_ = itr->second; + if (path_.empty()) { + exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname"); + } + } +} + +void ManifestDumpCommand::DoCommand() { + + std::string manifestfile; + + if (!path_.empty()) { + manifestfile = path_; + } else { + // We need to find the manifest file by searching the directory + // containing the db for files of the form MANIFEST_[0-9]+ + + std::vector<std::string> files; + Status s = options_.env->GetChildren(db_path_, &files); + if (!s.ok()) { + std::string err_msg = s.ToString(); + err_msg.append(": Failed to list the content of "); + err_msg.append(db_path_); + exec_state_ = LDBCommandExecuteResult::Failed(err_msg); + return; + } + const std::string kManifestNamePrefix = "MANIFEST-"; + std::string matched_file; +#ifdef OS_WIN + const char kPathDelim = '\\'; +#else + const char kPathDelim = '/'; +#endif + for (const auto& file_path : files) { + // Some Env::GetChildren() return absolute paths. Some directories' path + // end with path delim, e.g. '/' or '\\'. + size_t pos = file_path.find_last_of(kPathDelim); + if (pos == file_path.size() - 1) { + continue; + } + std::string fname; + if (pos != std::string::npos) { + // Absolute path. + fname.assign(file_path, pos + 1, file_path.size() - pos - 1); + } else { + fname = file_path; + } + uint64_t file_num = 0; + FileType file_type = kLogFile; // Just for initialization + if (ParseFileName(fname, &file_num, &file_type) && + file_type == kDescriptorFile) { + if (!matched_file.empty()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Multiple MANIFEST files found; use --path to select one"); + return; + } else { + matched_file.swap(fname); + } + } + } + if (matched_file.empty()) { + std::string err_msg("No MANIFEST found in "); + err_msg.append(db_path_); + exec_state_ = LDBCommandExecuteResult::Failed(err_msg); + return; + } + if (db_path_[db_path_.length() - 1] != '/') { + db_path_.append("/"); + } + manifestfile = db_path_ + matched_file; + } + + if (verbose_) { + fprintf(stdout, "Processing Manifest file %s\n", manifestfile.c_str()); + } + + DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_); + + if (verbose_) { + fprintf(stdout, "Processing Manifest file %s done\n", manifestfile.c_str()); + } +} + +// ---------------------------------------------------------------------------- +namespace { + +void GetLiveFilesChecksumInfoFromVersionSet(Options options, + const std::string& db_path, + FileChecksumList* checksum_list) { + EnvOptions sopt; + Status s; + std::string dbname(db_path); + std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10, + options.table_cache_numshardbits)); + // Notice we are using the default options not through SanitizeOptions(), + // if VersionSet::GetLiveFilesChecksumInfo depends on any option done by + // SanitizeOptions(), we need to initialize it manually. + options.db_paths.emplace_back(db_path, 0); + options.num_levels = 64; + WriteController wc(options.delayed_write_rate); + WriteBufferManager wb(options.db_write_buffer_size); + ImmutableDBOptions immutable_db_options(options); + VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc, + /*block_cache_tracer=*/nullptr); + std::vector<std::string> cf_name_list; + s = versions.ListColumnFamilies(&cf_name_list, db_path, + options.file_system.get()); + if (s.ok()) { + std::vector<ColumnFamilyDescriptor> cf_list; + for (const auto& name : cf_name_list) { + cf_list.emplace_back(name, ColumnFamilyOptions(options)); + } + s = versions.Recover(cf_list, true); + } + if (s.ok()) { + s = versions.GetLiveFilesChecksumInfo(checksum_list); + } + if (!s.ok()) { + fprintf(stderr, "Error Status: %s", s.ToString().c_str()); + } +} + +} // namespace + +const std::string FileChecksumDumpCommand::ARG_PATH = "path"; + +void FileChecksumDumpCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(FileChecksumDumpCommand::Name()); + ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]"); + ret.append("\n"); +} + +FileChecksumDumpCommand::FileChecksumDumpCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, BuildCmdLineOptions({ARG_PATH})), + path_("") { + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_PATH); + if (itr != options.end()) { + path_ = itr->second; + if (path_.empty()) { + exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname"); + } + } +} + +void FileChecksumDumpCommand::DoCommand() { + // print out the checksum information in the following format: + // sst file number, checksum function name, checksum value + // sst file number, checksum function name, checksum value + // ...... + + std::unique_ptr<FileChecksumList> checksum_list(NewFileChecksumList()); + GetLiveFilesChecksumInfoFromVersionSet(options_, db_path_, + checksum_list.get()); + if (checksum_list != nullptr) { + std::vector<uint64_t> file_numbers; + std::vector<std::string> checksums; + std::vector<std::string> checksum_func_names; + Status s = checksum_list->GetAllFileChecksums(&file_numbers, &checksums, + &checksum_func_names); + if (s.ok()) { + for (size_t i = 0; i < file_numbers.size(); i++) { + assert(i < file_numbers.size()); + assert(i < checksums.size()); + assert(i < checksum_func_names.size()); + fprintf(stdout, "%" PRId64 ", %s, %s\n", file_numbers[i], + checksum_func_names[i].c_str(), checksums[i].c_str()); + } + } + fprintf(stdout, "Print SST file checksum information finished \n"); + } +} + +// ---------------------------------------------------------------------------- + +void ListColumnFamiliesCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(ListColumnFamiliesCommand::Name()); + ret.append("\n"); +} + +ListColumnFamiliesCommand::ListColumnFamiliesCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {} + +void ListColumnFamiliesCommand::DoCommand() { + std::vector<std::string> column_families; + Status s = DB::ListColumnFamilies(options_, db_path_, &column_families); + if (!s.ok()) { + fprintf(stderr, "Error in processing db %s %s\n", db_path_.c_str(), + s.ToString().c_str()); + } else { + fprintf(stdout, "Column families in %s: \n{", db_path_.c_str()); + bool first = true; + for (auto cf : column_families) { + if (!first) { + fprintf(stdout, ", "); + } + first = false; + fprintf(stdout, "%s", cf.c_str()); + } + fprintf(stdout, "}\n"); + } +} + +void CreateColumnFamilyCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(CreateColumnFamilyCommand::Name()); + ret.append(" --db=<db_path> <new_column_family_name>"); + ret.append("\n"); +} + +CreateColumnFamilyCommand::CreateColumnFamilyCommand( + const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, true, {ARG_DB}) { + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::Failed( + "new column family name must be specified"); + } else { + new_cf_name_ = params[0]; + } +} + +void CreateColumnFamilyCommand::DoCommand() { + ColumnFamilyHandle* new_cf_handle = nullptr; + Status st = db_->CreateColumnFamily(options_, new_cf_name_, &new_cf_handle); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed( + "Fail to create new column family: " + st.ToString()); + } + delete new_cf_handle; + CloseDB(); +} + +void DropColumnFamilyCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(DropColumnFamilyCommand::Name()); + ret.append(" --db=<db_path> <column_family_name_to_drop>"); + ret.append("\n"); +} + +DropColumnFamilyCommand::DropColumnFamilyCommand( + const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, true, {ARG_DB}) { + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::Failed( + "The name of column family to drop must be specified"); + } else { + cf_name_to_drop_ = params[0]; + } +} + +void DropColumnFamilyCommand::DoCommand() { + auto iter = cf_handles_.find(cf_name_to_drop_); + if (iter == cf_handles_.end()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Column family: " + cf_name_to_drop_ + " doesn't exist in db."); + return; + } + ColumnFamilyHandle* cf_handle_to_drop = iter->second; + Status st = db_->DropColumnFamily(cf_handle_to_drop); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed( + "Fail to drop column family: " + st.ToString()); + } + CloseDB(); +} + +// ---------------------------------------------------------------------------- +namespace { + +// This function only called when it's the sane case of >1 buckets in time-range +// Also called only when timekv falls between ttl_start and ttl_end provided +void IncBucketCounts(std::vector<uint64_t>& bucket_counts, int ttl_start, + int time_range, int bucket_size, int timekv, + int num_buckets) { +#ifdef NDEBUG + (void)time_range; + (void)num_buckets; +#endif + assert(time_range > 0 && timekv >= ttl_start && bucket_size > 0 && + timekv < (ttl_start + time_range) && num_buckets > 1); + int bucket = (timekv - ttl_start) / bucket_size; + bucket_counts[bucket]++; +} + +void PrintBucketCounts(const std::vector<uint64_t>& bucket_counts, + int ttl_start, int ttl_end, int bucket_size, + int num_buckets) { + int time_point = ttl_start; + for(int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) { + fprintf(stdout, "Keys in range %s to %s : %lu\n", + TimeToHumanString(time_point).c_str(), + TimeToHumanString(time_point + bucket_size).c_str(), + (unsigned long)bucket_counts[i]); + } + fprintf(stdout, "Keys in range %s to %s : %lu\n", + TimeToHumanString(time_point).c_str(), + TimeToHumanString(ttl_end).c_str(), + (unsigned long)bucket_counts[num_buckets - 1]); +} + +} // namespace + +const std::string InternalDumpCommand::ARG_COUNT_ONLY = "count_only"; +const std::string InternalDumpCommand::ARG_COUNT_DELIM = "count_delim"; +const std::string InternalDumpCommand::ARG_STATS = "stats"; +const std::string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex"; + +InternalDumpCommand::InternalDumpCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, true, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, + ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, + ARG_COUNT_DELIM, ARG_STATS, ARG_INPUT_KEY_HEX})), + has_from_(false), + has_to_(false), + max_keys_(-1), + delim_("."), + count_only_(false), + count_delim_(false), + print_stats_(false), + is_input_key_hex_(false) { + has_from_ = ParseStringOption(options, ARG_FROM, &from_); + has_to_ = ParseStringOption(options, ARG_TO, &to_); + + ParseIntOption(options, ARG_MAX_KEYS, max_keys_, exec_state_); + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_COUNT_DELIM); + if (itr != options.end()) { + delim_ = itr->second; + count_delim_ = true; + // fprintf(stdout,"delim = %c\n",delim_[0]); + } else { + count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM); + delim_="."; + } + + print_stats_ = IsFlagPresent(flags, ARG_STATS); + count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY); + is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX); + + if (is_input_key_hex_) { + if (has_from_) { + from_ = HexToString(from_); + } + if (has_to_) { + to_ = HexToString(to_); + } + } +} + +void InternalDumpCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(InternalDumpCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append(" [--" + ARG_INPUT_KEY_HEX + "]"); + ret.append(" [--" + ARG_MAX_KEYS + "=<N>]"); + ret.append(" [--" + ARG_COUNT_ONLY + "]"); + ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]"); + ret.append(" [--" + ARG_STATS + "]"); + ret.append("\n"); +} + +void InternalDumpCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + + if (print_stats_) { + std::string stats; + if (db_->GetProperty(GetCfHandle(), "rocksdb.stats", &stats)) { + fprintf(stdout, "%s\n", stats.c_str()); + } + } + + // Cast as DBImpl to get internal iterator + std::vector<KeyVersion> key_versions; + Status st = GetAllKeyVersions(db_, GetCfHandle(), from_, to_, max_keys_, + &key_versions); + if (!st.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + return; + } + std::string rtype1, rtype2, row, val; + rtype2 = ""; + uint64_t c=0; + uint64_t s1=0,s2=0; + + long long count = 0; + for (auto& key_version : key_versions) { + InternalKey ikey(key_version.user_key, key_version.sequence, + static_cast<ValueType>(key_version.type)); + if (has_to_ && ikey.user_key() == to_) { + // GetAllKeyVersions() includes keys with user key `to_`, but idump has + // traditionally excluded such keys. + break; + } + ++count; + int k; + if (count_delim_) { + rtype1 = ""; + s1=0; + row = ikey.Encode().ToString(); + val = key_version.value; + for(k=0;row[k]!='\x01' && row[k]!='\0';k++) + s1++; + for(k=0;val[k]!='\x01' && val[k]!='\0';k++) + s1++; + for(int j=0;row[j]!=delim_[0] && row[j]!='\0' && row[j]!='\x01';j++) + rtype1+=row[j]; + if(rtype2.compare("") && rtype2.compare(rtype1)!=0) { + fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n", + rtype2.c_str(), c, s2); + c=1; + s2=s1; + rtype2 = rtype1; + } else { + c++; + s2+=s1; + rtype2=rtype1; + } + } + + if (!count_only_ && !count_delim_) { + std::string key = ikey.DebugString(is_key_hex_); + std::string value = Slice(key_version.value).ToString(is_value_hex_); + std::cout << key << " => " << value << "\n"; + } + + // Terminate if maximum number of keys have been dumped + if (max_keys_ > 0 && count >= max_keys_) break; + } + if(count_delim_) { + fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n", + rtype2.c_str(), c, s2); + } else { + fprintf(stdout, "Internal keys in range: %lld\n", count); + } +} + +const std::string DBDumperCommand::ARG_COUNT_ONLY = "count_only"; +const std::string DBDumperCommand::ARG_COUNT_DELIM = "count_delim"; +const std::string DBDumperCommand::ARG_STATS = "stats"; +const std::string DBDumperCommand::ARG_TTL_BUCKET = "bucket"; + +DBDumperCommand::DBDumperCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, true, + BuildCmdLineOptions( + {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, + ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM, + ARG_STATS, ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET, + ARG_TIMESTAMP, ARG_PATH})), + null_from_(true), + null_to_(true), + max_keys_(-1), + count_only_(false), + count_delim_(false), + print_stats_(false) { + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_FROM); + if (itr != options.end()) { + null_from_ = false; + from_ = itr->second; + } + + itr = options.find(ARG_TO); + if (itr != options.end()) { + null_to_ = false; + to_ = itr->second; + } + + itr = options.find(ARG_MAX_KEYS); + if (itr != options.end()) { + try { +#if defined(CYGWIN) + max_keys_ = strtol(itr->second.c_str(), 0, 10); +#else + max_keys_ = std::stoi(itr->second); +#endif + } catch (const std::invalid_argument&) { + exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS + + " has an invalid value"); + } catch (const std::out_of_range&) { + exec_state_ = LDBCommandExecuteResult::Failed( + ARG_MAX_KEYS + " has a value out-of-range"); + } + } + itr = options.find(ARG_COUNT_DELIM); + if (itr != options.end()) { + delim_ = itr->second; + count_delim_ = true; + } else { + count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM); + delim_="."; + } + + print_stats_ = IsFlagPresent(flags, ARG_STATS); + count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY); + + if (is_key_hex_) { + if (!null_from_) { + from_ = HexToString(from_); + } + if (!null_to_) { + to_ = HexToString(to_); + } + } + + itr = options.find(ARG_PATH); + if (itr != options.end()) { + path_ = itr->second; + if (db_path_.empty()) { + db_path_ = path_; + } + } +} + +void DBDumperCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(DBDumperCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append(" [--" + ARG_TTL + "]"); + ret.append(" [--" + ARG_MAX_KEYS + "=<N>]"); + ret.append(" [--" + ARG_TIMESTAMP + "]"); + ret.append(" [--" + ARG_COUNT_ONLY + "]"); + ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]"); + ret.append(" [--" + ARG_STATS + "]"); + ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]"); + ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]"); + ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]"); + ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]"); + ret.append("\n"); +} + +/** + * Handles two separate cases: + * + * 1) --db is specified - just dump the database. + * + * 2) --path is specified - determine based on file extension what dumping + * function to call. Please note that we intentionally use the extension + * and avoid probing the file contents under the assumption that renaming + * the files is not a supported scenario. + * + */ +void DBDumperCommand::DoCommand() { + if (!db_) { + assert(!path_.empty()); + std::string fileName = GetFileNameFromPath(path_); + uint64_t number; + FileType type; + + exec_state_ = LDBCommandExecuteResult::Succeed(""); + + if (!ParseFileName(fileName, &number, &type)) { + exec_state_ = + LDBCommandExecuteResult::Failed("Can't parse file type: " + path_); + return; + } + + switch (type) { + case kLogFile: + // TODO(myabandeh): allow configuring is_write_commited + DumpWalFile(options_, path_, /* print_header_ */ true, + /* print_values_ */ true, true /* is_write_commited */, + &exec_state_); + break; + case kTableFile: + DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true); + break; + case kDescriptorFile: + DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_, + /* json_ */ false); + break; + default: + exec_state_ = LDBCommandExecuteResult::Failed( + "File type not supported: " + path_); + break; + } + + } else { + DoDumpCommand(); + } +} + +void DBDumperCommand::DoDumpCommand() { + assert(nullptr != db_); + assert(path_.empty()); + + // Parse command line args + uint64_t count = 0; + if (print_stats_) { + std::string stats; + if (db_->GetProperty("rocksdb.stats", &stats)) { + fprintf(stdout, "%s\n", stats.c_str()); + } + } + + // Setup key iterator + ReadOptions scan_read_opts; + scan_read_opts.total_order_seek = true; + Iterator* iter = db_->NewIterator(scan_read_opts, GetCfHandle()); + Status st = iter->status(); + if (!st.ok()) { + exec_state_ = + LDBCommandExecuteResult::Failed("Iterator error." + st.ToString()); + } + + if (!null_from_) { + iter->Seek(from_); + } else { + iter->SeekToFirst(); + } + + int max_keys = max_keys_; + int ttl_start; + if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) { + ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time + } + int ttl_end; + if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) { + ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature + } + if (ttl_end < ttl_start) { + fprintf(stderr, "Error: End time can't be less than start time\n"); + delete iter; + return; + } + int time_range = ttl_end - ttl_start; + int bucket_size; + if (!ParseIntOption(option_map_, ARG_TTL_BUCKET, bucket_size, exec_state_) || + bucket_size <= 0) { + bucket_size = time_range; // Will have just 1 bucket by default + } + //cretaing variables for row count of each type + std::string rtype1, rtype2, row, val; + rtype2 = ""; + uint64_t c=0; + uint64_t s1=0,s2=0; + + // At this point, bucket_size=0 => time_range=0 + int num_buckets = (bucket_size >= time_range) + ? 1 + : ((time_range + bucket_size - 1) / bucket_size); + std::vector<uint64_t> bucket_counts(num_buckets, 0); + if (is_db_ttl_ && !count_only_ && timestamp_ && !count_delim_) { + fprintf(stdout, "Dumping key-values from %s to %s\n", + TimeToHumanString(ttl_start).c_str(), + TimeToHumanString(ttl_end).c_str()); + } + + HistogramImpl vsize_hist; + + for (; iter->Valid(); iter->Next()) { + int rawtime = 0; + // If end marker was specified, we stop before it + if (!null_to_ && (iter->key().ToString() >= to_)) + break; + // Terminate if maximum number of keys have been dumped + if (max_keys == 0) + break; + if (is_db_ttl_) { + TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(iter); + rawtime = it_ttl->timestamp(); + if (rawtime < ttl_start || rawtime >= ttl_end) { + continue; + } + } + if (max_keys > 0) { + --max_keys; + } + if (is_db_ttl_ && num_buckets > 1) { + IncBucketCounts(bucket_counts, ttl_start, time_range, bucket_size, + rawtime, num_buckets); + } + ++count; + if (count_delim_) { + rtype1 = ""; + row = iter->key().ToString(); + val = iter->value().ToString(); + s1 = row.size()+val.size(); + for(int j=0;row[j]!=delim_[0] && row[j]!='\0';j++) + rtype1+=row[j]; + if(rtype2.compare("") && rtype2.compare(rtype1)!=0) { + fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n", + rtype2.c_str(), c, s2); + c=1; + s2=s1; + rtype2 = rtype1; + } else { + c++; + s2+=s1; + rtype2=rtype1; + } + + } + + if (count_only_) { + vsize_hist.Add(iter->value().size()); + } + + if (!count_only_ && !count_delim_) { + if (is_db_ttl_ && timestamp_) { + fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str()); + } + std::string str = + PrintKeyValue(iter->key().ToString(), iter->value().ToString(), + is_key_hex_, is_value_hex_); + fprintf(stdout, "%s\n", str.c_str()); + } + } + + if (num_buckets > 1 && is_db_ttl_) { + PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size, + num_buckets); + } else if(count_delim_) { + fprintf(stdout, "%s => count:%" PRIu64 "\tsize:%" PRIu64 "\n", + rtype2.c_str(), c, s2); + } else { + fprintf(stdout, "Keys in range: %" PRIu64 "\n", count); + } + + if (count_only_) { + fprintf(stdout, "Value size distribution: \n"); + fprintf(stdout, "%s\n", vsize_hist.ToString().c_str()); + } + // Clean up + delete iter; +} + +const std::string ReduceDBLevelsCommand::ARG_NEW_LEVELS = "new_levels"; +const std::string ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS = + "print_old_levels"; + +ReduceDBLevelsCommand::ReduceDBLevelsCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_NEW_LEVELS, ARG_PRINT_OLD_LEVELS})), + old_levels_(1 << 7), + new_levels_(-1), + print_old_levels_(false) { + ParseIntOption(option_map_, ARG_NEW_LEVELS, new_levels_, exec_state_); + print_old_levels_ = IsFlagPresent(flags, ARG_PRINT_OLD_LEVELS); + + if(new_levels_ <= 0) { + exec_state_ = LDBCommandExecuteResult::Failed( + " Use --" + ARG_NEW_LEVELS + " to specify a new level number\n"); + } +} + +std::vector<std::string> ReduceDBLevelsCommand::PrepareArgs( + const std::string& db_path, int new_levels, bool print_old_level) { + std::vector<std::string> ret; + ret.push_back("reduce_levels"); + ret.push_back("--" + ARG_DB + "=" + db_path); + ret.push_back("--" + ARG_NEW_LEVELS + "=" + + ROCKSDB_NAMESPACE::ToString(new_levels)); + if(print_old_level) { + ret.push_back("--" + ARG_PRINT_OLD_LEVELS); + } + return ret; +} + +void ReduceDBLevelsCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(ReduceDBLevelsCommand::Name()); + ret.append(" --" + ARG_NEW_LEVELS + "=<New number of levels>"); + ret.append(" [--" + ARG_PRINT_OLD_LEVELS + "]"); + ret.append("\n"); +} + +Options ReduceDBLevelsCommand::PrepareOptionsForOpenDB() { + Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.num_levels = old_levels_; + opt.max_bytes_for_level_multiplier_additional.resize(opt.num_levels, 1); + // Disable size compaction + opt.max_bytes_for_level_base = 1ULL << 50; + opt.max_bytes_for_level_multiplier = 1; + return opt; +} + +Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt, + int* levels) { + ImmutableDBOptions db_options(opt); + EnvOptions soptions; + std::shared_ptr<Cache> tc( + NewLRUCache(opt.max_open_files - 10, opt.table_cache_numshardbits)); + const InternalKeyComparator cmp(opt.comparator); + WriteController wc(opt.delayed_write_rate); + WriteBufferManager wb(opt.db_write_buffer_size); + VersionSet versions(db_path_, &db_options, soptions, tc.get(), &wb, &wc, + /*block_cache_tracer=*/nullptr); + std::vector<ColumnFamilyDescriptor> dummy; + ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName, + ColumnFamilyOptions(opt)); + dummy.push_back(dummy_descriptor); + // We rely the VersionSet::Recover to tell us the internal data structures + // in the db. And the Recover() should never do any change + // (like LogAndApply) to the manifest file. + Status st = versions.Recover(dummy); + if (!st.ok()) { + return st; + } + int max = -1; + auto default_cfd = versions.GetColumnFamilySet()->GetDefault(); + for (int i = 0; i < default_cfd->NumberLevels(); i++) { + if (default_cfd->current()->storage_info()->NumLevelFiles(i)) { + max = i; + } + } + + *levels = max + 1; + return st; +} + +void ReduceDBLevelsCommand::DoCommand() { + if (new_levels_ <= 1) { + exec_state_ = + LDBCommandExecuteResult::Failed("Invalid number of levels.\n"); + return; + } + + Status st; + Options opt = PrepareOptionsForOpenDB(); + int old_level_num = -1; + opt.file_system.reset(new LegacyFileSystemWrapper(opt.env)); + ; + st = GetOldNumOfLevels(opt, &old_level_num); + if (!st.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + return; + } + + if (print_old_levels_) { + fprintf(stdout, "The old number of levels in use is %d\n", old_level_num); + } + + if (old_level_num <= new_levels_) { + return; + } + + old_levels_ = old_level_num; + + OpenDB(); + if (exec_state_.IsFailed()) { + return; + } + assert(db_ != nullptr); + // Compact the whole DB to put all files to the highest level. + fprintf(stdout, "Compacting the db...\n"); + db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr); + CloseDB(); + + EnvOptions soptions; + st = VersionSet::ReduceNumberOfLevels(db_path_, &opt, soptions, new_levels_); + if (!st.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + return; + } +} + +const std::string ChangeCompactionStyleCommand::ARG_OLD_COMPACTION_STYLE = + "old_compaction_style"; +const std::string ChangeCompactionStyleCommand::ARG_NEW_COMPACTION_STYLE = + "new_compaction_style"; + +ChangeCompactionStyleCommand::ChangeCompactionStyleCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions( + {ARG_OLD_COMPACTION_STYLE, ARG_NEW_COMPACTION_STYLE})), + old_compaction_style_(-1), + new_compaction_style_(-1) { + ParseIntOption(option_map_, ARG_OLD_COMPACTION_STYLE, old_compaction_style_, + exec_state_); + if (old_compaction_style_ != kCompactionStyleLevel && + old_compaction_style_ != kCompactionStyleUniversal) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Use --" + ARG_OLD_COMPACTION_STYLE + " to specify old compaction " + + "style. Check ldb help for proper compaction style value.\n"); + return; + } + + ParseIntOption(option_map_, ARG_NEW_COMPACTION_STYLE, new_compaction_style_, + exec_state_); + if (new_compaction_style_ != kCompactionStyleLevel && + new_compaction_style_ != kCompactionStyleUniversal) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Use --" + ARG_NEW_COMPACTION_STYLE + " to specify new compaction " + + "style. Check ldb help for proper compaction style value.\n"); + return; + } + + if (new_compaction_style_ == old_compaction_style_) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Old compaction style is the same as new compaction style. " + "Nothing to do.\n"); + return; + } + + if (old_compaction_style_ == kCompactionStyleUniversal && + new_compaction_style_ == kCompactionStyleLevel) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Convert from universal compaction to level compaction. " + "Nothing to do.\n"); + return; + } +} + +void ChangeCompactionStyleCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(ChangeCompactionStyleCommand::Name()); + ret.append(" --" + ARG_OLD_COMPACTION_STYLE + "=<Old compaction style: 0 " + + "for level compaction, 1 for universal compaction>"); + ret.append(" --" + ARG_NEW_COMPACTION_STYLE + "=<New compaction style: 0 " + + "for level compaction, 1 for universal compaction>"); + ret.append("\n"); +} + +Options ChangeCompactionStyleCommand::PrepareOptionsForOpenDB() { + Options opt = LDBCommand::PrepareOptionsForOpenDB(); + + if (old_compaction_style_ == kCompactionStyleLevel && + new_compaction_style_ == kCompactionStyleUniversal) { + // In order to convert from level compaction to universal compaction, we + // need to compact all data into a single file and move it to level 0. + opt.disable_auto_compactions = true; + opt.target_file_size_base = INT_MAX; + opt.target_file_size_multiplier = 1; + opt.max_bytes_for_level_base = INT_MAX; + opt.max_bytes_for_level_multiplier = 1; + } + + return opt; +} + +void ChangeCompactionStyleCommand::DoCommand() { + // print db stats before we have made any change + std::string property; + std::string files_per_level; + for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) { + db_->GetProperty(GetCfHandle(), + "rocksdb.num-files-at-level" + NumberToString(i), + &property); + + // format print string + char buf[100]; + snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str()); + files_per_level += buf; + } + fprintf(stdout, "files per level before compaction: %s\n", + files_per_level.c_str()); + + // manual compact into a single file and move the file to level 0 + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 0; + db_->CompactRange(compact_options, GetCfHandle(), nullptr, nullptr); + + // verify compaction result + files_per_level = ""; + int num_files = 0; + for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) { + db_->GetProperty(GetCfHandle(), + "rocksdb.num-files-at-level" + NumberToString(i), + &property); + + // format print string + char buf[100]; + snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str()); + files_per_level += buf; + + num_files = atoi(property.c_str()); + + // level 0 should have only 1 file + if (i == 0 && num_files != 1) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Number of db files at " + "level 0 after compaction is " + + ToString(num_files) + ", not 1.\n"); + return; + } + // other levels should have no file + if (i > 0 && num_files != 0) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Number of db files at " + "level " + + ToString(i) + " after compaction is " + ToString(num_files) + + ", not 0.\n"); + return; + } + } + + fprintf(stdout, "files per level after compaction: %s\n", + files_per_level.c_str()); +} + +// ---------------------------------------------------------------------------- + +namespace { + +struct StdErrReporter : public log::Reader::Reporter { + void Corruption(size_t /*bytes*/, const Status& s) override { + std::cerr << "Corruption detected in log file " << s.ToString() << "\n"; + } +}; + +class InMemoryHandler : public WriteBatch::Handler { + public: + InMemoryHandler(std::stringstream& row, bool print_values, + bool write_after_commit = false) + : Handler(), + row_(row), + print_values_(print_values), + write_after_commit_(write_after_commit) {} + + void commonPutMerge(const Slice& key, const Slice& value) { + std::string k = LDBCommand::StringToHex(key.ToString()); + if (print_values_) { + std::string v = LDBCommand::StringToHex(value.ToString()); + row_ << k << " : "; + row_ << v << " "; + } else { + row_ << k << " "; + } + } + + Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override { + row_ << "PUT(" << cf << ") : "; + commonPutMerge(key, value); + return Status::OK(); + } + + Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override { + row_ << "MERGE(" << cf << ") : "; + commonPutMerge(key, value); + return Status::OK(); + } + + Status MarkNoop(bool) override { + row_ << "NOOP "; + return Status::OK(); + } + + Status DeleteCF(uint32_t cf, const Slice& key) override { + row_ << "DELETE(" << cf << ") : "; + row_ << LDBCommand::StringToHex(key.ToString()) << " "; + return Status::OK(); + } + + Status SingleDeleteCF(uint32_t cf, const Slice& key) override { + row_ << "SINGLE_DELETE(" << cf << ") : "; + row_ << LDBCommand::StringToHex(key.ToString()) << " "; + return Status::OK(); + } + + Status DeleteRangeCF(uint32_t cf, const Slice& begin_key, + const Slice& end_key) override { + row_ << "DELETE_RANGE(" << cf << ") : "; + row_ << LDBCommand::StringToHex(begin_key.ToString()) << " "; + row_ << LDBCommand::StringToHex(end_key.ToString()) << " "; + return Status::OK(); + } + + Status MarkBeginPrepare(bool unprepare) override { + row_ << "BEGIN_PREPARE("; + row_ << (unprepare ? "true" : "false") << ") "; + return Status::OK(); + } + + Status MarkEndPrepare(const Slice& xid) override { + row_ << "END_PREPARE("; + row_ << LDBCommand::StringToHex(xid.ToString()) << ") "; + return Status::OK(); + } + + Status MarkRollback(const Slice& xid) override { + row_ << "ROLLBACK("; + row_ << LDBCommand::StringToHex(xid.ToString()) << ") "; + return Status::OK(); + } + + Status MarkCommit(const Slice& xid) override { + row_ << "COMMIT("; + row_ << LDBCommand::StringToHex(xid.ToString()) << ") "; + return Status::OK(); + } + + ~InMemoryHandler() override {} + + protected: + bool WriteAfterCommit() const override { return write_after_commit_; } + + private: + std::stringstream& row_; + bool print_values_; + bool write_after_commit_; +}; + +void DumpWalFile(Options options, std::string wal_file, bool print_header, + bool print_values, bool is_write_committed, + LDBCommandExecuteResult* exec_state) { + Env* env = options.env; + EnvOptions soptions(options); + std::unique_ptr<SequentialFileReader> wal_file_reader; + + Status status; + { + std::unique_ptr<SequentialFile> file; + status = env->NewSequentialFile(wal_file, &file, soptions); + if (status.ok()) { + wal_file_reader.reset(new SequentialFileReader( + NewLegacySequentialFileWrapper(file), wal_file)); + } + } + if (!status.ok()) { + if (exec_state) { + *exec_state = LDBCommandExecuteResult::Failed("Failed to open WAL file " + + status.ToString()); + } else { + std::cerr << "Error: Failed to open WAL file " << status.ToString() + << std::endl; + } + } else { + StdErrReporter reporter; + uint64_t log_number; + FileType type; + + // we need the log number, but ParseFilename expects dbname/NNN.log. + std::string sanitized = wal_file; + size_t lastslash = sanitized.rfind('/'); + if (lastslash != std::string::npos) + sanitized = sanitized.substr(lastslash + 1); + if (!ParseFileName(sanitized, &log_number, &type)) { + // bogus input, carry on as best we can + log_number = 0; + } + log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter, + true /* checksum */, log_number); + std::string scratch; + WriteBatch batch; + Slice record; + std::stringstream row; + if (print_header) { + std::cout << "Sequence,Count,ByteSize,Physical Offset,Key(s)"; + if (print_values) { + std::cout << " : value "; + } + std::cout << "\n"; + } + while (reader.ReadRecord(&record, &scratch)) { + row.str(""); + if (record.size() < WriteBatchInternal::kHeader) { + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); + } else { + WriteBatchInternal::SetContents(&batch, record); + row << WriteBatchInternal::Sequence(&batch) << ","; + row << WriteBatchInternal::Count(&batch) << ","; + row << WriteBatchInternal::ByteSize(&batch) << ","; + row << reader.LastRecordOffset() << ","; + InMemoryHandler handler(row, print_values, is_write_committed); + batch.Iterate(&handler); + row << "\n"; + } + std::cout << row.str(); + } + } +} + +} // namespace + +const std::string WALDumperCommand::ARG_WAL_FILE = "walfile"; +const std::string WALDumperCommand::ARG_WRITE_COMMITTED = "write_committed"; +const std::string WALDumperCommand::ARG_PRINT_VALUE = "print_value"; +const std::string WALDumperCommand::ARG_PRINT_HEADER = "header"; + +WALDumperCommand::WALDumperCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, true, + BuildCmdLineOptions({ARG_WAL_FILE, ARG_WRITE_COMMITTED, + ARG_PRINT_HEADER, ARG_PRINT_VALUE})), + print_header_(false), + print_values_(false), + is_write_committed_(false) { + wal_file_.clear(); + + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_WAL_FILE); + if (itr != options.end()) { + wal_file_ = itr->second; + } + + + print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER); + print_values_ = IsFlagPresent(flags, ARG_PRINT_VALUE); + is_write_committed_ = ParseBooleanOption(options, ARG_WRITE_COMMITTED, true); + + if (wal_file_.empty()) { + exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE + + " must be specified."); + } +} + +void WALDumperCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(WALDumperCommand::Name()); + ret.append(" --" + ARG_WAL_FILE + "=<write_ahead_log_file_path>"); + ret.append(" [--" + ARG_PRINT_HEADER + "] "); + ret.append(" [--" + ARG_PRINT_VALUE + "] "); + ret.append(" [--" + ARG_WRITE_COMMITTED + "=true|false] "); + ret.append("\n"); +} + +void WALDumperCommand::DoCommand() { + DumpWalFile(options_, wal_file_, print_header_, print_values_, + is_write_committed_, &exec_state_); +} + +// ---------------------------------------------------------------------------- + +GetCommand::GetCommand(const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, true, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::Failed( + "<key> must be specified for the get command"); + } else { + key_ = params.at(0); + } + + if (is_key_hex_) { + key_ = HexToString(key_); + } +} + +void GetCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(GetCommand::Name()); + ret.append(" <key>"); + ret.append(" [--" + ARG_TTL + "]"); + ret.append("\n"); +} + +void GetCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + std::string value; + Status st = db_->Get(ReadOptions(), GetCfHandle(), key_, &value); + if (st.ok()) { + fprintf(stdout, "%s\n", + (is_value_hex_ ? StringToHex(value) : value).c_str()); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +// ---------------------------------------------------------------------------- + +ApproxSizeCommand::ApproxSizeCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, true, + BuildCmdLineOptions( + {ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO})) { + if (options.find(ARG_FROM) != options.end()) { + start_key_ = options.find(ARG_FROM)->second; + } else { + exec_state_ = LDBCommandExecuteResult::Failed( + ARG_FROM + " must be specified for approxsize command"); + return; + } + + if (options.find(ARG_TO) != options.end()) { + end_key_ = options.find(ARG_TO)->second; + } else { + exec_state_ = LDBCommandExecuteResult::Failed( + ARG_TO + " must be specified for approxsize command"); + return; + } + + if (is_key_hex_) { + start_key_ = HexToString(start_key_); + end_key_ = HexToString(end_key_); + } +} + +void ApproxSizeCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(ApproxSizeCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append("\n"); +} + +void ApproxSizeCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + Range ranges[1]; + ranges[0] = Range(start_key_, end_key_); + uint64_t sizes[1]; + db_->GetApproximateSizes(GetCfHandle(), ranges, 1, sizes); + fprintf(stdout, "%lu\n", (unsigned long)sizes[0]); + /* Weird that GetApproximateSizes() returns void, although documentation + * says that it returns a Status object. + if (!st.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } + */ +} + +// ---------------------------------------------------------------------------- + +BatchPutCommand::BatchPutCommand( + const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) { + if (params.size() < 2) { + exec_state_ = LDBCommandExecuteResult::Failed( + "At least one <key> <value> pair must be specified batchput."); + } else if (params.size() % 2 != 0) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Equal number of <key>s and <value>s must be specified for batchput."); + } else { + for (size_t i = 0; i < params.size(); i += 2) { + std::string key = params.at(i); + std::string value = params.at(i + 1); + key_values_.push_back(std::pair<std::string, std::string>( + is_key_hex_ ? HexToString(key) : key, + is_value_hex_ ? HexToString(value) : value)); + } + } + create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); +} + +void BatchPutCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(BatchPutCommand::Name()); + ret.append(" <key> <value> [<key> <value>] [..]"); + ret.append(" [--" + ARG_TTL + "]"); + ret.append("\n"); +} + +void BatchPutCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + WriteBatch batch; + + for (std::vector<std::pair<std::string, std::string>>::const_iterator itr = + key_values_.begin(); + itr != key_values_.end(); ++itr) { + batch.Put(GetCfHandle(), itr->first, itr->second); + } + Status st = db_->Write(WriteOptions(), &batch); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +Options BatchPutCommand::PrepareOptionsForOpenDB() { + Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.create_if_missing = create_if_missing_; + return opt; +} + +// ---------------------------------------------------------------------------- + +ScanCommand::ScanCommand(const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, true, + BuildCmdLineOptions({ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX, + ARG_TO, ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP, + ARG_MAX_KEYS, ARG_TTL_START, ARG_TTL_END})), + start_key_specified_(false), + end_key_specified_(false), + max_keys_scanned_(-1), + no_value_(false) { + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_FROM); + if (itr != options.end()) { + start_key_ = itr->second; + if (is_key_hex_) { + start_key_ = HexToString(start_key_); + } + start_key_specified_ = true; + } + itr = options.find(ARG_TO); + if (itr != options.end()) { + end_key_ = itr->second; + if (is_key_hex_) { + end_key_ = HexToString(end_key_); + } + end_key_specified_ = true; + } + + std::vector<std::string>::const_iterator vitr = + std::find(flags.begin(), flags.end(), ARG_NO_VALUE); + if (vitr != flags.end()) { + no_value_ = true; + } + + itr = options.find(ARG_MAX_KEYS); + if (itr != options.end()) { + try { +#if defined(CYGWIN) + max_keys_scanned_ = strtol(itr->second.c_str(), 0, 10); +#else + max_keys_scanned_ = std::stoi(itr->second); +#endif + } catch (const std::invalid_argument&) { + exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS + + " has an invalid value"); + } catch (const std::out_of_range&) { + exec_state_ = LDBCommandExecuteResult::Failed( + ARG_MAX_KEYS + " has a value out-of-range"); + } + } +} + +void ScanCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(ScanCommand::Name()); + ret.append(HelpRangeCmdArgs()); + ret.append(" [--" + ARG_TTL + "]"); + ret.append(" [--" + ARG_TIMESTAMP + "]"); + ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] "); + ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]"); + ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]"); + ret.append(" [--" + ARG_NO_VALUE + "]"); + ret.append("\n"); +} + +void ScanCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + + int num_keys_scanned = 0; + ReadOptions scan_read_opts; + scan_read_opts.total_order_seek = true; + Iterator* it = db_->NewIterator(scan_read_opts, GetCfHandle()); + if (start_key_specified_) { + it->Seek(start_key_); + } else { + it->SeekToFirst(); + } + int ttl_start; + if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) { + ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time + } + int ttl_end; + if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) { + ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature + } + if (ttl_end < ttl_start) { + fprintf(stderr, "Error: End time can't be less than start time\n"); + delete it; + return; + } + if (is_db_ttl_ && timestamp_) { + fprintf(stdout, "Scanning key-values from %s to %s\n", + TimeToHumanString(ttl_start).c_str(), + TimeToHumanString(ttl_end).c_str()); + } + for ( ; + it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_); + it->Next()) { + if (is_db_ttl_) { + TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(it); + int rawtime = it_ttl->timestamp(); + if (rawtime < ttl_start || rawtime >= ttl_end) { + continue; + } + if (timestamp_) { + fprintf(stdout, "%s ", TimeToHumanString(rawtime).c_str()); + } + } + + Slice key_slice = it->key(); + + std::string formatted_key; + if (is_key_hex_) { + formatted_key = "0x" + key_slice.ToString(true /* hex */); + key_slice = formatted_key; + } else if (ldb_options_.key_formatter) { + formatted_key = ldb_options_.key_formatter->Format(key_slice); + key_slice = formatted_key; + } + + if (no_value_) { + fprintf(stdout, "%.*s\n", static_cast<int>(key_slice.size()), + key_slice.data()); + } else { + Slice val_slice = it->value(); + std::string formatted_value; + if (is_value_hex_) { + formatted_value = "0x" + val_slice.ToString(true /* hex */); + val_slice = formatted_value; + } + fprintf(stdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()), + key_slice.data(), static_cast<int>(val_slice.size()), + val_slice.data()); + } + + num_keys_scanned++; + if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) { + break; + } + } + if (!it->status().ok()) { // Check for any errors found during the scan + exec_state_ = LDBCommandExecuteResult::Failed(it->status().ToString()); + } + delete it; +} + +// ---------------------------------------------------------------------------- + +DeleteCommand::DeleteCommand(const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::Failed( + "KEY must be specified for the delete command"); + } else { + key_ = params.at(0); + if (is_key_hex_) { + key_ = HexToString(key_); + } + } +} + +void DeleteCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(DeleteCommand::Name() + " <key>"); + ret.append("\n"); +} + +void DeleteCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + Status st = db_->Delete(WriteOptions(), GetCfHandle(), key_); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +DeleteRangeCommand::DeleteRangeCommand( + const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + if (params.size() != 2) { + exec_state_ = LDBCommandExecuteResult::Failed( + "begin and end keys must be specified for the delete command"); + } else { + begin_key_ = params.at(0); + end_key_ = params.at(1); + if (is_key_hex_) { + begin_key_ = HexToString(begin_key_); + end_key_ = HexToString(end_key_); + } + } +} + +void DeleteRangeCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(DeleteRangeCommand::Name() + " <begin key> <end key>"); + ret.append("\n"); +} + +void DeleteRangeCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + Status st = + db_->DeleteRange(WriteOptions(), GetCfHandle(), begin_key_, end_key_); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +PutCommand::PutCommand(const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) { + if (params.size() != 2) { + exec_state_ = LDBCommandExecuteResult::Failed( + "<key> and <value> must be specified for the put command"); + } else { + key_ = params.at(0); + value_ = params.at(1); + } + + if (is_key_hex_) { + key_ = HexToString(key_); + } + + if (is_value_hex_) { + value_ = HexToString(value_); + } + create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING); +} + +void PutCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(PutCommand::Name()); + ret.append(" <key> <value> "); + ret.append(" [--" + ARG_TTL + "]"); + ret.append("\n"); +} + +void PutCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + Status st = db_->Put(WriteOptions(), GetCfHandle(), key_, value_); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +Options PutCommand::PrepareOptionsForOpenDB() { + Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.create_if_missing = create_if_missing_; + return opt; +} + +// ---------------------------------------------------------------------------- + +const char* DBQuerierCommand::HELP_CMD = "help"; +const char* DBQuerierCommand::GET_CMD = "get"; +const char* DBQuerierCommand::PUT_CMD = "put"; +const char* DBQuerierCommand::DELETE_CMD = "delete"; + +DBQuerierCommand::DBQuerierCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, false, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + +} + +void DBQuerierCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(DBQuerierCommand::Name()); + ret.append(" [--" + ARG_TTL + "]"); + ret.append("\n"); + ret.append(" Starts a REPL shell. Type help for list of available " + "commands."); + ret.append("\n"); +} + +void DBQuerierCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + + ReadOptions read_options; + WriteOptions write_options; + + std::string line; + std::string key; + std::string value; + while (getline(std::cin, line, '\n')) { + // Parse line into std::vector<std::string> + std::vector<std::string> tokens; + size_t pos = 0; + while (true) { + size_t pos2 = line.find(' ', pos); + if (pos2 == std::string::npos) { + break; + } + tokens.push_back(line.substr(pos, pos2-pos)); + pos = pos2 + 1; + } + tokens.push_back(line.substr(pos)); + + const std::string& cmd = tokens[0]; + + if (cmd == HELP_CMD) { + fprintf(stdout, + "get <key>\n" + "put <key> <value>\n" + "delete <key>\n"); + } else if (cmd == DELETE_CMD && tokens.size() == 2) { + key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]); + db_->Delete(write_options, GetCfHandle(), Slice(key)); + fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str()); + } else if (cmd == PUT_CMD && tokens.size() == 3) { + key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]); + value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]); + db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value)); + fprintf(stdout, "Successfully put %s %s\n", + tokens[1].c_str(), tokens[2].c_str()); + } else if (cmd == GET_CMD && tokens.size() == 2) { + key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]); + if (db_->Get(read_options, GetCfHandle(), Slice(key), &value).ok()) { + fprintf(stdout, "%s\n", PrintKeyValue(key, value, + is_key_hex_, is_value_hex_).c_str()); + } else { + fprintf(stdout, "Not found %s\n", tokens[1].c_str()); + } + } else { + fprintf(stdout, "Unknown command %s\n", line.c_str()); + } + } +} + +// ---------------------------------------------------------------------------- + +CheckConsistencyCommand::CheckConsistencyCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {} + +void CheckConsistencyCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(CheckConsistencyCommand::Name()); + ret.append("\n"); +} + +void CheckConsistencyCommand::DoCommand() { + Options opt = PrepareOptionsForOpenDB(); + opt.paranoid_checks = true; + if (!exec_state_.IsNotStarted()) { + return; + } + DB* db; + Status st = DB::OpenForReadOnly(opt, db_path_, &db, false); + delete db; + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +// ---------------------------------------------------------------------------- + +const std::string CheckPointCommand::ARG_CHECKPOINT_DIR = "checkpoint_dir"; + +CheckPointCommand::CheckPointCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false /* is_read_only */, + BuildCmdLineOptions({ARG_CHECKPOINT_DIR})) { + auto itr = options.find(ARG_CHECKPOINT_DIR); + if (itr != options.end()) { + checkpoint_dir_ = itr->second; + } +} + +void CheckPointCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(CheckPointCommand::Name()); + ret.append(" [--" + ARG_CHECKPOINT_DIR + "] "); + ret.append("\n"); +} + +void CheckPointCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + Checkpoint* checkpoint; + Status status = Checkpoint::Create(db_, &checkpoint); + status = checkpoint->CreateCheckpoint(checkpoint_dir_); + if (status.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(status.ToString()); + } +} + +// ---------------------------------------------------------------------------- + +RepairCommand::RepairCommand(const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {} + +void RepairCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(RepairCommand::Name()); + ret.append("\n"); +} + +void RepairCommand::DoCommand() { + Options options = PrepareOptionsForOpenDB(); + options.info_log.reset(new StderrLogger(InfoLogLevel::WARN_LEVEL)); + Status status = RepairDB(db_path_, options); + if (status.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(status.ToString()); + } +} + +// ---------------------------------------------------------------------------- + +const std::string BackupableCommand::ARG_NUM_THREADS = "num_threads"; +const std::string BackupableCommand::ARG_BACKUP_ENV_URI = "backup_env_uri"; +const std::string BackupableCommand::ARG_BACKUP_DIR = "backup_dir"; +const std::string BackupableCommand::ARG_STDERR_LOG_LEVEL = "stderr_log_level"; + +BackupableCommand::BackupableCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, false /* is_read_only */, + BuildCmdLineOptions({ARG_BACKUP_ENV_URI, ARG_BACKUP_DIR, + ARG_NUM_THREADS, ARG_STDERR_LOG_LEVEL})), + num_threads_(1) { + auto itr = options.find(ARG_NUM_THREADS); + if (itr != options.end()) { + num_threads_ = std::stoi(itr->second); + } + itr = options.find(ARG_BACKUP_ENV_URI); + if (itr != options.end()) { + backup_env_uri_ = itr->second; + } + itr = options.find(ARG_BACKUP_DIR); + if (itr == options.end()) { + exec_state_ = LDBCommandExecuteResult::Failed("--" + ARG_BACKUP_DIR + + ": missing backup directory"); + } else { + backup_dir_ = itr->second; + } + + itr = options.find(ARG_STDERR_LOG_LEVEL); + if (itr != options.end()) { + int stderr_log_level = std::stoi(itr->second); + if (stderr_log_level < 0 || + stderr_log_level >= InfoLogLevel::NUM_INFO_LOG_LEVELS) { + exec_state_ = LDBCommandExecuteResult::Failed( + ARG_STDERR_LOG_LEVEL + " must be >= 0 and < " + + std::to_string(InfoLogLevel::NUM_INFO_LOG_LEVELS) + "."); + } else { + logger_.reset( + new StderrLogger(static_cast<InfoLogLevel>(stderr_log_level))); + } + } +} + +void BackupableCommand::Help(const std::string& name, std::string& ret) { + ret.append(" "); + ret.append(name); + ret.append(" [--" + ARG_BACKUP_ENV_URI + "] "); + ret.append(" [--" + ARG_BACKUP_DIR + "] "); + ret.append(" [--" + ARG_NUM_THREADS + "] "); + ret.append(" [--" + ARG_STDERR_LOG_LEVEL + "=<int (InfoLogLevel)>] "); + ret.append("\n"); +} + +// ---------------------------------------------------------------------------- + +BackupCommand::BackupCommand(const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : BackupableCommand(params, options, flags) {} + +void BackupCommand::Help(std::string& ret) { + BackupableCommand::Help(Name(), ret); +} + +void BackupCommand::DoCommand() { + BackupEngine* backup_engine; + Status status; + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + fprintf(stdout, "open db OK\n"); + Env* custom_env = nullptr; + Env::LoadEnv(backup_env_uri_, &custom_env, &backup_env_guard_); + assert(custom_env != nullptr); + + BackupableDBOptions backup_options = + BackupableDBOptions(backup_dir_, custom_env); + backup_options.info_log = logger_.get(); + backup_options.max_background_operations = num_threads_; + status = BackupEngine::Open(custom_env, backup_options, &backup_engine); + if (status.ok()) { + fprintf(stdout, "open backup engine OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(status.ToString()); + return; + } + status = backup_engine->CreateNewBackup(db_); + if (status.ok()) { + fprintf(stdout, "create new backup OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(status.ToString()); + return; + } +} + +// ---------------------------------------------------------------------------- + +RestoreCommand::RestoreCommand( + const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : BackupableCommand(params, options, flags) {} + +void RestoreCommand::Help(std::string& ret) { + BackupableCommand::Help(Name(), ret); +} + +void RestoreCommand::DoCommand() { + Env* custom_env = nullptr; + Env::LoadEnv(backup_env_uri_, &custom_env, &backup_env_guard_); + assert(custom_env != nullptr); + + std::unique_ptr<BackupEngineReadOnly> restore_engine; + Status status; + { + BackupableDBOptions opts(backup_dir_, custom_env); + opts.info_log = logger_.get(); + opts.max_background_operations = num_threads_; + BackupEngineReadOnly* raw_restore_engine_ptr; + status = + BackupEngineReadOnly::Open(custom_env, opts, &raw_restore_engine_ptr); + if (status.ok()) { + restore_engine.reset(raw_restore_engine_ptr); + } + } + if (status.ok()) { + fprintf(stdout, "open restore engine OK\n"); + status = restore_engine->RestoreDBFromLatestBackup(db_path_, db_path_); + } + if (status.ok()) { + fprintf(stdout, "restore from backup OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(status.ToString()); + } +} + +// ---------------------------------------------------------------------------- + +namespace { + +void DumpSstFile(Options options, std::string filename, bool output_hex, + bool show_properties) { + std::string from_key; + std::string to_key; + if (filename.length() <= 4 || + filename.rfind(".sst") != filename.length() - 4) { + std::cout << "Invalid sst file name." << std::endl; + return; + } + // no verification + // TODO: add support for decoding blob indexes in ldb as well + ROCKSDB_NAMESPACE::SstFileDumper dumper( + options, filename, /* verify_checksum */ false, output_hex, + /* decode_blob_index */ false); + Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(), + false, // has_from + from_key, false, // has_to + to_key); + if (!st.ok()) { + std::cerr << "Error in reading SST file " << filename << st.ToString() + << std::endl; + return; + } + + if (show_properties) { + const ROCKSDB_NAMESPACE::TableProperties* table_properties; + + std::shared_ptr<const ROCKSDB_NAMESPACE::TableProperties> + table_properties_from_reader; + st = dumper.ReadTableProperties(&table_properties_from_reader); + if (!st.ok()) { + std::cerr << filename << ": " << st.ToString() + << ". Try to use initial table properties" << std::endl; + table_properties = dumper.GetInitTableProperties(); + } else { + table_properties = table_properties_from_reader.get(); + } + if (table_properties != nullptr) { + std::cout << std::endl << "Table Properties:" << std::endl; + std::cout << table_properties->ToString("\n") << std::endl; + } + } +} + +} // namespace + +DBFileDumperCommand::DBFileDumperCommand( + const std::vector<std::string>& /*params*/, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {} + +void DBFileDumperCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(DBFileDumperCommand::Name()); + ret.append("\n"); +} + +void DBFileDumperCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + Status s; + + std::cout << "Manifest File" << std::endl; + std::cout << "==============================" << std::endl; + std::string manifest_filename; + s = ReadFileToString(db_->GetEnv(), CurrentFileName(db_->GetName()), + &manifest_filename); + if (!s.ok() || manifest_filename.empty() || + manifest_filename.back() != '\n') { + std::cerr << "Error when reading CURRENT file " + << CurrentFileName(db_->GetName()) << std::endl; + } + // remove the trailing '\n' + manifest_filename.resize(manifest_filename.size() - 1); + std::string manifest_filepath = db_->GetName() + "/" + manifest_filename; + std::cout << manifest_filepath << std::endl; + DumpManifestFile(options_, manifest_filepath, false, false, false); + std::cout << std::endl; + + std::cout << "SST Files" << std::endl; + std::cout << "==============================" << std::endl; + std::vector<LiveFileMetaData> metadata; + db_->GetLiveFilesMetaData(&metadata); + for (auto& fileMetadata : metadata) { + std::string filename = fileMetadata.db_path + fileMetadata.name; + std::cout << filename << " level:" << fileMetadata.level << std::endl; + std::cout << "------------------------------" << std::endl; + DumpSstFile(options_, filename, false, true); + std::cout << std::endl; + } + std::cout << std::endl; + + std::cout << "Write Ahead Log Files" << std::endl; + std::cout << "==============================" << std::endl; + ROCKSDB_NAMESPACE::VectorLogPtr wal_files; + s = db_->GetSortedWalFiles(wal_files); + if (!s.ok()) { + std::cerr << "Error when getting WAL files" << std::endl; + } else { + for (auto& wal : wal_files) { + // TODO(qyang): option.wal_dir should be passed into ldb command + std::string filename = db_->GetOptions().wal_dir + wal->PathName(); + std::cout << filename << std::endl; + // TODO(myabandeh): allow configuring is_write_commited + DumpWalFile(options_, filename, true, true, true /* is_write_commited */, + &exec_state_); + } + } +} + +void WriteExternalSstFilesCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(WriteExternalSstFilesCommand::Name()); + ret.append(" <output_sst_path>"); + ret.append("\n"); +} + +WriteExternalSstFilesCommand::WriteExternalSstFilesCommand( + const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, false /* is_read_only */, + BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, + ARG_TO, ARG_CREATE_IF_MISSING})) { + create_if_missing_ = + IsFlagPresent(flags, ARG_CREATE_IF_MISSING) || + ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false); + if (params.size() != 1) { + exec_state_ = LDBCommandExecuteResult::Failed( + "output SST file path must be specified"); + } else { + output_sst_path_ = params.at(0); + } +} + +void WriteExternalSstFilesCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + ColumnFamilyHandle* cfh = GetCfHandle(); + SstFileWriter sst_file_writer(EnvOptions(), db_->GetOptions(), cfh); + Status status = sst_file_writer.Open(output_sst_path_); + if (!status.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed("failed to open SST file: " + + status.ToString()); + return; + } + + int bad_lines = 0; + std::string line; + std::ifstream ifs_stdin("/dev/stdin"); + std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin; + while (getline(*istream_p, line, '\n')) { + std::string key; + std::string value; + if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) { + status = sst_file_writer.Put(key, value); + if (!status.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "failed to write record to file: " + status.ToString()); + return; + } + } else if (0 == line.find("Keys in range:")) { + // ignore this line + } else if (0 == line.find("Created bg thread 0x")) { + // ignore this line + } else { + bad_lines++; + } + } + + status = sst_file_writer.Finish(); + if (!status.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Failed to finish writing to file: " + status.ToString()); + return; + } + + if (bad_lines > 0) { + fprintf(stderr, "Warning: %d bad lines ignored.\n", bad_lines); + } + exec_state_ = LDBCommandExecuteResult::Succeed( + "external SST file written to " + output_sst_path_); +} + +Options WriteExternalSstFilesCommand::PrepareOptionsForOpenDB() { + Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.create_if_missing = create_if_missing_; + return opt; +} + +const std::string IngestExternalSstFilesCommand::ARG_MOVE_FILES = "move_files"; +const std::string IngestExternalSstFilesCommand::ARG_SNAPSHOT_CONSISTENCY = + "snapshot_consistency"; +const std::string IngestExternalSstFilesCommand::ARG_ALLOW_GLOBAL_SEQNO = + "allow_global_seqno"; +const std::string IngestExternalSstFilesCommand::ARG_ALLOW_BLOCKING_FLUSH = + "allow_blocking_flush"; +const std::string IngestExternalSstFilesCommand::ARG_INGEST_BEHIND = + "ingest_behind"; +const std::string IngestExternalSstFilesCommand::ARG_WRITE_GLOBAL_SEQNO = + "write_global_seqno"; + +void IngestExternalSstFilesCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(IngestExternalSstFilesCommand::Name()); + ret.append(" <input_sst_path>"); + ret.append(" [--" + ARG_MOVE_FILES + "] "); + ret.append(" [--" + ARG_SNAPSHOT_CONSISTENCY + "] "); + ret.append(" [--" + ARG_ALLOW_GLOBAL_SEQNO + "] "); + ret.append(" [--" + ARG_ALLOW_BLOCKING_FLUSH + "] "); + ret.append(" [--" + ARG_INGEST_BEHIND + "] "); + ret.append(" [--" + ARG_WRITE_GLOBAL_SEQNO + "] "); + ret.append("\n"); +} + +IngestExternalSstFilesCommand::IngestExternalSstFilesCommand( + const std::vector<std::string>& params, + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand( + options, flags, false /* is_read_only */, + BuildCmdLineOptions({ARG_MOVE_FILES, ARG_SNAPSHOT_CONSISTENCY, + ARG_ALLOW_GLOBAL_SEQNO, ARG_CREATE_IF_MISSING, + ARG_ALLOW_BLOCKING_FLUSH, ARG_INGEST_BEHIND, + ARG_WRITE_GLOBAL_SEQNO})), + move_files_(false), + snapshot_consistency_(true), + allow_global_seqno_(true), + allow_blocking_flush_(true), + ingest_behind_(false), + write_global_seqno_(true) { + create_if_missing_ = + IsFlagPresent(flags, ARG_CREATE_IF_MISSING) || + ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false); + move_files_ = IsFlagPresent(flags, ARG_MOVE_FILES) || + ParseBooleanOption(options, ARG_MOVE_FILES, false); + snapshot_consistency_ = + IsFlagPresent(flags, ARG_SNAPSHOT_CONSISTENCY) || + ParseBooleanOption(options, ARG_SNAPSHOT_CONSISTENCY, true); + allow_global_seqno_ = + IsFlagPresent(flags, ARG_ALLOW_GLOBAL_SEQNO) || + ParseBooleanOption(options, ARG_ALLOW_GLOBAL_SEQNO, true); + allow_blocking_flush_ = + IsFlagPresent(flags, ARG_ALLOW_BLOCKING_FLUSH) || + ParseBooleanOption(options, ARG_ALLOW_BLOCKING_FLUSH, true); + ingest_behind_ = IsFlagPresent(flags, ARG_INGEST_BEHIND) || + ParseBooleanOption(options, ARG_INGEST_BEHIND, false); + write_global_seqno_ = + IsFlagPresent(flags, ARG_WRITE_GLOBAL_SEQNO) || + ParseBooleanOption(options, ARG_WRITE_GLOBAL_SEQNO, true); + + if (allow_global_seqno_) { + if (!write_global_seqno_) { + fprintf(stderr, + "Warning: not writing global_seqno to the ingested SST can\n" + "prevent older versions of RocksDB from being able to open it\n"); + } + } else { + if (write_global_seqno_) { + exec_state_ = LDBCommandExecuteResult::Failed( + "ldb cannot write global_seqno to the ingested SST when global_seqno " + "is not allowed"); + } + } + + if (params.size() != 1) { + exec_state_ = + LDBCommandExecuteResult::Failed("input SST path must be specified"); + } else { + input_sst_path_ = params.at(0); + } +} + +void IngestExternalSstFilesCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + if (GetExecuteState().IsFailed()) { + return; + } + ColumnFamilyHandle* cfh = GetCfHandle(); + IngestExternalFileOptions ifo; + ifo.move_files = move_files_; + ifo.snapshot_consistency = snapshot_consistency_; + ifo.allow_global_seqno = allow_global_seqno_; + ifo.allow_blocking_flush = allow_blocking_flush_; + ifo.ingest_behind = ingest_behind_; + ifo.write_global_seqno = write_global_seqno_; + Status status = db_->IngestExternalFile(cfh, {input_sst_path_}, ifo); + if (!status.ok()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "failed to ingest external SST: " + status.ToString()); + } else { + exec_state_ = + LDBCommandExecuteResult::Succeed("external SST files ingested"); + } +} + +Options IngestExternalSstFilesCommand::PrepareOptionsForOpenDB() { + Options opt = LDBCommand::PrepareOptionsForOpenDB(); + opt.create_if_missing = create_if_missing_; + return opt; +} + +ListFileRangeDeletesCommand::ListFileRangeDeletesCommand( + const std::map<std::string, std::string>& options, + const std::vector<std::string>& flags) + : LDBCommand(options, flags, true, BuildCmdLineOptions({ARG_MAX_KEYS})) { + std::map<std::string, std::string>::const_iterator itr = + options.find(ARG_MAX_KEYS); + if (itr != options.end()) { + try { +#if defined(CYGWIN) + max_keys_ = strtol(itr->second.c_str(), 0, 10); +#else + max_keys_ = std::stoi(itr->second); +#endif + } catch (const std::invalid_argument&) { + exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS + + " has an invalid value"); + } catch (const std::out_of_range&) { + exec_state_ = LDBCommandExecuteResult::Failed( + ARG_MAX_KEYS + " has a value out-of-range"); + } + } +} + +void ListFileRangeDeletesCommand::Help(std::string& ret) { + ret.append(" "); + ret.append(ListFileRangeDeletesCommand::Name()); + ret.append(" [--" + ARG_MAX_KEYS + "=<N>]"); + ret.append(" : print tombstones in SST files.\n"); +} + +void ListFileRangeDeletesCommand::DoCommand() { + if (!db_) { + assert(GetExecuteState().IsFailed()); + return; + } + + DBImpl* db_impl = static_cast_with_check<DBImpl, DB>(db_->GetRootDB()); + + std::string out_str; + + Status st = + db_impl->TablesRangeTombstoneSummary(GetCfHandle(), max_keys_, &out_str); + if (st.ok()) { + TEST_SYNC_POINT_CALLBACK( + "ListFileRangeDeletesCommand::DoCommand:BeforePrint", &out_str); + fprintf(stdout, "%s\n", out_str.c_str()); + } else { + exec_state_ = LDBCommandExecuteResult::Failed(st.ToString()); + } +} + +} // namespace ROCKSDB_NAMESPACE +#endif // ROCKSDB_LITE |