summaryrefslogtreecommitdiffstats
path: root/src/rocksdb/tools/dump
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/rocksdb/tools/dump/db_dump_tool.cc260
-rw-r--r--src/rocksdb/tools/dump/rocksdb_dump.cc63
-rw-r--r--src/rocksdb/tools/dump/rocksdb_undump.cc62
3 files changed, 385 insertions, 0 deletions
diff --git a/src/rocksdb/tools/dump/db_dump_tool.cc b/src/rocksdb/tools/dump/db_dump_tool.cc
new file mode 100644
index 000000000..427a54d99
--- /dev/null
+++ b/src/rocksdb/tools/dump/db_dump_tool.cc
@@ -0,0 +1,260 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include "rocksdb/db_dump_tool.h"
+
+#include <cinttypes>
+#include <iostream>
+
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "util/coding.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+bool DbDumpTool::Run(const DumpOptions& dump_options,
+ ROCKSDB_NAMESPACE::Options options) {
+ ROCKSDB_NAMESPACE::DB* dbptr;
+ ROCKSDB_NAMESPACE::Status status;
+ std::unique_ptr<ROCKSDB_NAMESPACE::WritableFile> dumpfile;
+ char hostname[1024];
+ int64_t timesec = 0;
+ std::string abspath;
+ char json[4096];
+
+ static const char* magicstr = "ROCKDUMP";
+ static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
+
+ ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();
+
+ // Open the database
+ options.create_if_missing = false;
+ status = ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, dump_options.db_path,
+ &dbptr);
+ if (!status.ok()) {
+ std::cerr << "Unable to open database '" << dump_options.db_path
+ << "' for reading: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ const std::unique_ptr<ROCKSDB_NAMESPACE::DB> db(dbptr);
+
+ status = env->NewWritableFile(dump_options.dump_location, &dumpfile,
+ ROCKSDB_NAMESPACE::EnvOptions());
+ if (!status.ok()) {
+ std::cerr << "Unable to open dump file '" << dump_options.dump_location
+ << "' for writing: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ ROCKSDB_NAMESPACE::Slice magicslice(magicstr, 8);
+ status = dumpfile->Append(magicslice);
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ ROCKSDB_NAMESPACE::Slice versionslice(versionstr, 8);
+ status = dumpfile->Append(versionslice);
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ if (dump_options.anonymous) {
+ snprintf(json, sizeof(json), "{}");
+ } else {
+ status = env->GetHostName(hostname, sizeof(hostname));
+ status = env->GetCurrentTime(&timesec);
+ status = env->GetAbsolutePath(dump_options.db_path, &abspath);
+ snprintf(json, sizeof(json),
+ "{ \"database-path\": \"%s\", \"hostname\": \"%s\", "
+ "\"creation-time\": %" PRIi64 " }",
+ abspath.c_str(), hostname, timesec);
+ }
+
+ ROCKSDB_NAMESPACE::Slice infoslice(json, strlen(json));
+ char infosize[4];
+ ROCKSDB_NAMESPACE::EncodeFixed32(infosize, (uint32_t)infoslice.size());
+ ROCKSDB_NAMESPACE::Slice infosizeslice(infosize, 4);
+ status = dumpfile->Append(infosizeslice);
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+ status = dumpfile->Append(infoslice);
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ const std::unique_ptr<ROCKSDB_NAMESPACE::Iterator> it(
+ db->NewIterator(ROCKSDB_NAMESPACE::ReadOptions()));
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ char keysize[4];
+ ROCKSDB_NAMESPACE::EncodeFixed32(keysize, (uint32_t)it->key().size());
+ ROCKSDB_NAMESPACE::Slice keysizeslice(keysize, 4);
+ status = dumpfile->Append(keysizeslice);
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+ status = dumpfile->Append(it->key());
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ char valsize[4];
+ ROCKSDB_NAMESPACE::EncodeFixed32(valsize, (uint32_t)it->value().size());
+ ROCKSDB_NAMESPACE::Slice valsizeslice(valsize, 4);
+ status = dumpfile->Append(valsizeslice);
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+ status = dumpfile->Append(it->value());
+ if (!status.ok()) {
+ std::cerr << "Append failed: " << status.ToString() << std::endl;
+ return false;
+ }
+ }
+ if (!it->status().ok()) {
+ std::cerr << "Database iteration failed: " << status.ToString()
+ << std::endl;
+ return false;
+ }
+ return true;
+}
+
+bool DbUndumpTool::Run(const UndumpOptions& undump_options,
+ ROCKSDB_NAMESPACE::Options options) {
+ ROCKSDB_NAMESPACE::DB* dbptr;
+ ROCKSDB_NAMESPACE::Status status;
+ ROCKSDB_NAMESPACE::Env* env;
+ std::unique_ptr<ROCKSDB_NAMESPACE::SequentialFile> dumpfile;
+ ROCKSDB_NAMESPACE::Slice slice;
+ char scratch8[8];
+
+ static const char* magicstr = "ROCKDUMP";
+ static const char versionstr[8] = {0, 0, 0, 0, 0, 0, 0, 1};
+
+ env = ROCKSDB_NAMESPACE::Env::Default();
+
+ status = env->NewSequentialFile(undump_options.dump_location, &dumpfile,
+ ROCKSDB_NAMESPACE::EnvOptions());
+ if (!status.ok()) {
+ std::cerr << "Unable to open dump file '" << undump_options.dump_location
+ << "' for reading: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ status = dumpfile->Read(8, &slice, scratch8);
+ if (!status.ok() || slice.size() != 8 ||
+ memcmp(slice.data(), magicstr, 8) != 0) {
+ std::cerr << "File '" << undump_options.dump_location
+ << "' is not a recognizable dump file." << std::endl;
+ return false;
+ }
+
+ status = dumpfile->Read(8, &slice, scratch8);
+ if (!status.ok() || slice.size() != 8 ||
+ memcmp(slice.data(), versionstr, 8) != 0) {
+ std::cerr << "File '" << undump_options.dump_location
+ << "' version not recognized." << std::endl;
+ return false;
+ }
+
+ status = dumpfile->Read(4, &slice, scratch8);
+ if (!status.ok() || slice.size() != 4) {
+ std::cerr << "Unable to read info blob size." << std::endl;
+ return false;
+ }
+ uint32_t infosize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
+ status = dumpfile->Skip(infosize);
+ if (!status.ok()) {
+ std::cerr << "Unable to skip info blob: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ options.create_if_missing = true;
+ status = ROCKSDB_NAMESPACE::DB::Open(options, undump_options.db_path, &dbptr);
+ if (!status.ok()) {
+ std::cerr << "Unable to open database '" << undump_options.db_path
+ << "' for writing: " << status.ToString() << std::endl;
+ return false;
+ }
+
+ const std::unique_ptr<ROCKSDB_NAMESPACE::DB> db(dbptr);
+
+ uint32_t last_keysize = 64;
+ size_t last_valsize = 1 << 20;
+ std::unique_ptr<char[]> keyscratch(new char[last_keysize]);
+ std::unique_ptr<char[]> valscratch(new char[last_valsize]);
+
+ while (1) {
+ uint32_t keysize, valsize;
+ ROCKSDB_NAMESPACE::Slice keyslice;
+ ROCKSDB_NAMESPACE::Slice valslice;
+
+ status = dumpfile->Read(4, &slice, scratch8);
+ if (!status.ok() || slice.size() != 4) break;
+ keysize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
+ if (keysize > last_keysize) {
+ while (keysize > last_keysize) last_keysize *= 2;
+ keyscratch = std::unique_ptr<char[]>(new char[last_keysize]);
+ }
+
+ status = dumpfile->Read(keysize, &keyslice, keyscratch.get());
+ if (!status.ok() || keyslice.size() != keysize) {
+ std::cerr << "Key read failure: "
+ << (status.ok() ? "insufficient data" : status.ToString())
+ << std::endl;
+ return false;
+ }
+
+ status = dumpfile->Read(4, &slice, scratch8);
+ if (!status.ok() || slice.size() != 4) {
+ std::cerr << "Unable to read value size: "
+ << (status.ok() ? "insufficient data" : status.ToString())
+ << std::endl;
+ return false;
+ }
+ valsize = ROCKSDB_NAMESPACE::DecodeFixed32(slice.data());
+ if (valsize > last_valsize) {
+ while (valsize > last_valsize) last_valsize *= 2;
+ valscratch = std::unique_ptr<char[]>(new char[last_valsize]);
+ }
+
+ status = dumpfile->Read(valsize, &valslice, valscratch.get());
+ if (!status.ok() || valslice.size() != valsize) {
+ std::cerr << "Unable to read value: "
+ << (status.ok() ? "insufficient data" : status.ToString())
+ << std::endl;
+ return false;
+ }
+
+ status = db->Put(ROCKSDB_NAMESPACE::WriteOptions(), keyslice, valslice);
+ if (!status.ok()) {
+ fprintf(stderr, "Unable to write database entry\n");
+ return false;
+ }
+ }
+
+ if (undump_options.compact_db) {
+ status = db->CompactRange(ROCKSDB_NAMESPACE::CompactRangeOptions(), nullptr,
+ nullptr);
+ if (!status.ok()) {
+ fprintf(stderr,
+ "Unable to compact the database after loading the dumped file\n");
+ return false;
+ }
+ }
+ return true;
+}
+} // namespace ROCKSDB_NAMESPACE
+#endif // ROCKSDB_LITE
diff --git a/src/rocksdb/tools/dump/rocksdb_dump.cc b/src/rocksdb/tools/dump/rocksdb_dump.cc
new file mode 100644
index 000000000..358457e92
--- /dev/null
+++ b/src/rocksdb/tools/dump/rocksdb_dump.cc
@@ -0,0 +1,63 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#if !(defined GFLAGS) || defined(ROCKSDB_LITE)
+
+#include <cstdio>
+int main() {
+#ifndef GFLAGS
+ fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+#endif
+#ifdef ROCKSDB_LITE
+ fprintf(stderr, "DbDumpTool is not supported in ROCKSDB_LITE\n");
+#endif
+ return 1;
+}
+
+#else
+
+#include "rocksdb/convenience.h"
+#include "rocksdb/db_dump_tool.h"
+#include "util/gflags_compat.h"
+
+DEFINE_string(db_path, "", "Path to the db that will be dumped");
+DEFINE_string(dump_location, "", "Path to where the dump file location");
+DEFINE_bool(anonymous, false,
+ "Remove information like db path, creation time from dumped file");
+DEFINE_string(db_options, "",
+ "Options string used to open the database that will be dumped");
+
+int main(int argc, char** argv) {
+ GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+ if (FLAGS_db_path == "" || FLAGS_dump_location == "") {
+ fprintf(stderr, "Please set --db_path and --dump_location\n");
+ return 1;
+ }
+
+ ROCKSDB_NAMESPACE::DumpOptions dump_options;
+ dump_options.db_path = FLAGS_db_path;
+ dump_options.dump_location = FLAGS_dump_location;
+ dump_options.anonymous = FLAGS_anonymous;
+
+ ROCKSDB_NAMESPACE::Options db_options;
+ if (FLAGS_db_options != "") {
+ ROCKSDB_NAMESPACE::Options parsed_options;
+ ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::GetOptionsFromString(
+ db_options, FLAGS_db_options, &parsed_options);
+ if (!s.ok()) {
+ fprintf(stderr, "Cannot parse provided db_options\n");
+ return 1;
+ }
+ db_options = parsed_options;
+ }
+
+ ROCKSDB_NAMESPACE::DbDumpTool tool;
+ if (!tool.Run(dump_options, db_options)) {
+ return 1;
+ }
+ return 0;
+}
+#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
diff --git a/src/rocksdb/tools/dump/rocksdb_undump.cc b/src/rocksdb/tools/dump/rocksdb_undump.cc
new file mode 100644
index 000000000..2ff128548
--- /dev/null
+++ b/src/rocksdb/tools/dump/rocksdb_undump.cc
@@ -0,0 +1,62 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#if !(defined GFLAGS) || defined(ROCKSDB_LITE)
+
+#include <cstdio>
+int main() {
+#ifndef GFLAGS
+ fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+#endif
+#ifdef ROCKSDB_LITE
+ fprintf(stderr, "DbUndumpTool is not supported in ROCKSDB_LITE\n");
+#endif
+ return 1;
+}
+
+#else
+
+#include "rocksdb/convenience.h"
+#include "rocksdb/db_dump_tool.h"
+#include "util/gflags_compat.h"
+
+DEFINE_string(dump_location, "", "Path to the dump file that will be loaded");
+DEFINE_string(db_path, "", "Path to the db that we will undump the file into");
+DEFINE_bool(compact, false, "Compact the db after loading the dumped file");
+DEFINE_string(db_options, "",
+ "Options string used to open the database that will be loaded");
+
+int main(int argc, char **argv) {
+ GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
+
+ if (FLAGS_db_path == "" || FLAGS_dump_location == "") {
+ fprintf(stderr, "Please set --db_path and --dump_location\n");
+ return 1;
+ }
+
+ ROCKSDB_NAMESPACE::UndumpOptions undump_options;
+ undump_options.db_path = FLAGS_db_path;
+ undump_options.dump_location = FLAGS_dump_location;
+ undump_options.compact_db = FLAGS_compact;
+
+ ROCKSDB_NAMESPACE::Options db_options;
+ if (FLAGS_db_options != "") {
+ ROCKSDB_NAMESPACE::Options parsed_options;
+ ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::GetOptionsFromString(
+ db_options, FLAGS_db_options, &parsed_options);
+ if (!s.ok()) {
+ fprintf(stderr, "Cannot parse provided db_options\n");
+ return 1;
+ }
+ db_options = parsed_options;
+ }
+
+ ROCKSDB_NAMESPACE::DbUndumpTool tool;
+ if (!tool.Run(undump_options, db_options)) {
+ return 1;
+ }
+ return 0;
+}
+#endif // !(defined GFLAGS) || defined(ROCKSDB_LITE)
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-23 23:31:10 +0000