summaryrefslogtreecommitdiffstats
path: root/storage/tokudb/PerconaFT/tools
diff options
context:
space:
mode:
Diffstat (limited to 'storage/tokudb/PerconaFT/tools')
-rw-r--r--storage/tokudb/PerconaFT/tools/CMakeLists.txt25
-rw-r--r--storage/tokudb/PerconaFT/tools/ftverify.cc452
-rw-r--r--storage/tokudb/PerconaFT/tools/pmprof31
-rw-r--r--storage/tokudb/PerconaFT/tools/tdb-recover.cc80
-rw-r--r--storage/tokudb/PerconaFT/tools/tokudb_dump.cc685
-rw-r--r--storage/tokudb/PerconaFT/tools/tokuft_logprint.cc74
-rw-r--r--storage/tokudb/PerconaFT/tools/tokuftdump.cc1246
7 files changed, 2593 insertions, 0 deletions
diff --git a/storage/tokudb/PerconaFT/tools/CMakeLists.txt b/storage/tokudb/PerconaFT/tools/CMakeLists.txt
new file mode 100644
index 00000000..dd54249a
--- /dev/null
+++ b/storage/tokudb/PerconaFT/tools/CMakeLists.txt
@@ -0,0 +1,25 @@
+set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE DONT_DEPRECATE_ERRNO)
+
+set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify)
+foreach(tool ${tools})
+ add_executable(${tool} ${tool}.cc)
+ add_dependencies(${tool} install_tdb_h)
+ target_link_libraries(${tool} ${LIBTOKUDB}_static ft_static z lzma snappy ${LIBTOKUPORTABILITY}_static ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_SYSTEM_LIBS})
+
+ # detect when we are being built as a subproject
+ if (DEFINED MYSQL_PROJECT_NAME_DOCSTRING)
+ if ((CMAKE_BUILD_TYPE MATCHES "Debug") AND
+ (CMAKE_CXX_FLAGS_DEBUG MATCHES " -DENABLED_DEBUG_SYNC"))
+ target_link_libraries(${tool} sql)
+ endif()
+ target_link_libraries(${tool} mysys)
+ endif ()
+
+ add_space_separated_property(TARGET ${tool} COMPILE_FLAGS -fvisibility=hidden)
+endforeach(tool)
+
+# link in math.h library just for this tool.
+target_link_libraries(ftverify m)
+
+install(TARGETS tokuftdump DESTINATION ${INSTALL_BINDIR} COMPONENT tokudb-engine)
+install(TARGETS tokuft_logprint DESTINATION ${INSTALL_BINDIR} COMPONENT tokudb-engine)
diff --git a/storage/tokudb/PerconaFT/tools/ftverify.cc b/storage/tokudb/PerconaFT/tools/ftverify.cc
new file mode 100644
index 00000000..ee40b991
--- /dev/null
+++ b/storage/tokudb/PerconaFT/tools/ftverify.cc
@@ -0,0 +1,452 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+////////////////////////////////////////////////////////////////////
+// ftverify - Command line tool that checks the validity of a given
+// fractal tree file, one block at a time.
+////////////////////////////////////////////////////////////////////
+
+#include "portability/toku_assert.h"
+#include "portability/toku_list.h"
+#include "portability/toku_portability.h"
+
+#include "ft/serialize/block_allocator.h"
+#include "ft/ft-internal.h"
+#include "ft/serialize/ft-serialize.h"
+#include "ft/serialize/ft_layout_version.h"
+#include "ft/serialize/ft_node-serialize.h"
+#include "ft/node.h"
+#include "ft/serialize/rbuf.h"
+#include "ft/serialize/sub_block.h"
+#include "util/threadpool.h"
+
+#include <fcntl.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sysexits.h>
+#include <unistd.h>
+
+static int num_cores = 0; // cache the number of cores for the parallelization
+static struct toku_thread_pool *ft_pool = NULL;
+static FILE *outf;
+static double pct = 0.5;
+
+// Struct for reporting sub block stats.
+struct verify_block_extra {
+ BLOCKNUM b;
+ int n_sub_blocks;
+ uint32_t header_length;
+ uint32_t calc_xsum;
+ uint32_t stored_xsum;
+ bool header_valid;
+ bool sub_blocks_valid;
+ struct sub_block_info *sub_block_results;
+};
+
+// Initialization function for the sub block stats.
+static void
+init_verify_block_extra(BLOCKNUM b, struct verify_block_extra *e)
+{
+ static const struct verify_block_extra default_vbe =
+ {
+ .b = { 0 },
+ .n_sub_blocks = 0,
+ .header_length = 0,
+ .calc_xsum = 0,
+ .stored_xsum = 0,
+ .header_valid = true,
+ .sub_blocks_valid = true,
+ .sub_block_results = NULL
+ };
+ *e = default_vbe;
+ e->b = b;
+}
+
+// Reports percentage of completed blocks.
+static void
+report(int64_t blocks_done, int64_t blocks_failed, int64_t total_blocks)
+{
+ int64_t blocks_per_report = llrint(pct * total_blocks / 100.0);
+ if (blocks_per_report < 1) {
+ blocks_per_report = 1;
+ }
+ if (blocks_done % blocks_per_report == 0) {
+ double pct_actually_done = (100.0 * blocks_done) / total_blocks;
+ printf("% 3.3lf%% | %" PRId64 " blocks checked, %" PRId64 " bad block(s) detected\n",
+ pct_actually_done, blocks_done, blocks_failed);
+ fflush(stdout);
+ }
+}
+
+// Helper function to deserialize one of the two headers for the ft
+// we are checking.
+static void
+deserialize_headers(int fd, struct ft **h1p, struct ft **h2p)
+{
+ struct rbuf rb_0;
+ struct rbuf rb_1;
+ uint64_t checkpoint_count_0;
+ uint64_t checkpoint_count_1;
+ LSN checkpoint_lsn_0;
+ LSN checkpoint_lsn_1;
+ uint32_t version_0, version_1;
+ bool h0_acceptable = false;
+ bool h1_acceptable = false;
+ int r0, r1;
+ int r;
+
+ {
+ toku_off_t header_0_off = 0;
+ r0 = deserialize_ft_from_fd_into_rbuf(
+ fd,
+ header_0_off,
+ &rb_0,
+ &checkpoint_count_0,
+ &checkpoint_lsn_0,
+ &version_0
+ );
+ if ((r0==0) && (checkpoint_lsn_0.lsn <= MAX_LSN.lsn)) {
+ h0_acceptable = true;
+ }
+ }
+ {
+ toku_off_t header_1_off = BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE;
+ r1 = deserialize_ft_from_fd_into_rbuf(
+ fd,
+ header_1_off,
+ &rb_1,
+ &checkpoint_count_1,
+ &checkpoint_lsn_1,
+ &version_1
+ );
+ if ((r1==0) && (checkpoint_lsn_1.lsn <= MAX_LSN.lsn)) {
+ h1_acceptable = true;
+ }
+ }
+
+ // If either header is too new, the dictionary is unreadable
+ if (r0 == TOKUDB_DICTIONARY_TOO_NEW || r1 == TOKUDB_DICTIONARY_TOO_NEW) {
+ fprintf(stderr, "This dictionary was created with a version of PerconaFT that is too new. Aborting.\n");
+ abort();
+ }
+ if (h0_acceptable) {
+ printf("Found dictionary header 1 with LSN %" PRIu64 "\n", checkpoint_lsn_0.lsn);
+ r = deserialize_ft_versioned(fd, &rb_0, h1p, version_0);
+
+ if (r != 0) {
+ printf("---Header Error----\n");
+ }
+
+ } else {
+ *h1p = NULL;
+ }
+ if (h1_acceptable) {
+ printf("Found dictionary header 2 with LSN %" PRIu64 "\n", checkpoint_lsn_1.lsn);
+ r = deserialize_ft_versioned(fd, &rb_1, h2p, version_1);
+ if (r != 0) {
+ printf("---Header Error----\n");
+ }
+ } else {
+ *h2p = NULL;
+ }
+
+ if (rb_0.buf) toku_free(rb_0.buf);
+ if (rb_1.buf) toku_free(rb_1.buf);
+}
+
+// Helper struct for tracking block checking progress.
+struct check_block_table_extra {
+ int fd;
+ int64_t blocks_done, blocks_failed, total_blocks;
+ struct ft *h;
+};
+
+// Check non-upgraded (legacy) node.
+// NOTE: These nodes have less checksumming than more
+// recent nodes. This effectively means that we are
+// skipping over these nodes.
+static int
+check_old_node(FTNODE node, struct rbuf *rb, int version)
+{
+ int r = 0;
+ read_legacy_node_info(node, rb, version);
+ // For version 14 nodes, advance the buffer to the end
+ // and verify the checksum.
+ if (version == FT_FIRST_LAYOUT_VERSION_WITH_END_TO_END_CHECKSUM) {
+ // Advance the buffer to the end.
+ rb->ndone = rb->size - 4;
+ r = check_legacy_end_checksum(rb);
+ }
+
+ return r;
+}
+
+// Read, decompress, and check the given block.
+static int
+check_block(BLOCKNUM blocknum, int64_t UU(blocksize), int64_t UU(address), void *extra)
+{
+ int r = 0;
+ int failure = 0;
+ struct check_block_table_extra *CAST_FROM_VOIDP(cbte, extra);
+ int fd = cbte->fd;
+ FT ft = cbte->h;
+
+ struct verify_block_extra be;
+ init_verify_block_extra(blocknum, &be);
+
+ // Let's read the block off of disk and fill a buffer with that
+ // block.
+ struct rbuf rb = RBUF_INITIALIZER;
+ read_block_from_fd_into_rbuf(fd, blocknum, ft, &rb);
+
+ // Allocate the node.
+ FTNODE XMALLOC(node);
+
+ initialize_ftnode(node, blocknum);
+
+ r = read_and_check_magic(&rb);
+ if (r == DB_BADFORMAT) {
+ printf(" Magic failed.\n");
+ failure++;
+ }
+
+ r = read_and_check_version(node, &rb);
+ if (r != 0) {
+ printf(" Version check failed.\n");
+ failure++;
+ }
+
+ int version = node->layout_version_read_from_disk;
+
+ ////////////////////////////
+ // UPGRADE FORK GOES HERE //
+ ////////////////////////////
+
+ // Check nodes before major layout changes in version 15.
+ // All newer versions should follow the same layout, for now.
+ // This predicate would need to be changed if the layout
+ // of the nodes on disk does indeed change in the future.
+ if (version < FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES)
+ {
+ struct rbuf nrb;
+ // Use old decompression method for legacy nodes.
+ r = decompress_from_raw_block_into_rbuf(rb.buf, rb.size, &nrb, blocknum);
+ if (r != 0) {
+ failure++;
+ goto cleanup;
+ }
+
+ // Check the end-to-end checksum.
+ r = check_old_node(node, &nrb, version);
+ if (r != 0) {
+ failure++;
+ }
+ goto cleanup;
+ }
+
+ read_node_info(node, &rb, version);
+
+ FTNODE_DISK_DATA ndd;
+ allocate_and_read_partition_offsets(node, &rb, &ndd);
+
+ r = check_node_info_checksum(&rb);
+ if (r == TOKUDB_BAD_CHECKSUM) {
+ printf(" Node info checksum failed.\n");
+ failure++;
+ }
+
+ // Get the partition info sub block.
+ struct sub_block sb;
+ sub_block_init(&sb);
+ r = read_compressed_sub_block(&rb, &sb);
+ if (r != 0) {
+ printf(" Partition info checksum failed.\n");
+ failure++;
+ }
+
+ just_decompress_sub_block(&sb);
+
+ // If we want to inspect the data inside the partitions, we need
+ // to call setup_ftnode_partitions(node, bfe, true)
+
+ // <CER> TODO: Create function for this.
+ // Using the node info, decompress all the keys and pivots to
+ // detect any corruptions.
+ for (int i = 0; i < node->n_children; ++i) {
+ uint32_t curr_offset = BP_START(ndd,i);
+ uint32_t curr_size = BP_SIZE(ndd,i);
+ struct rbuf curr_rbuf = {.buf = NULL, .size = 0, .ndone = 0};
+ rbuf_init(&curr_rbuf, rb.buf + curr_offset, curr_size);
+ struct sub_block curr_sb;
+ sub_block_init(&curr_sb);
+
+ r = read_compressed_sub_block(&rb, &sb);
+ if (r != 0) {
+ printf(" Compressed child partition %d checksum failed.\n", i);
+ failure++;
+ }
+ just_decompress_sub_block(&sb);
+
+ r = verify_ftnode_sub_block(&sb, nullptr, blocknum);
+ if (r != 0) {
+ printf(" Uncompressed child partition %d checksum failed.\n", i);
+ failure++;
+ }
+
+ // <CER> If needed, we can print row and/or pivot info at this
+ // point.
+ }
+
+cleanup:
+ // Cleanup and error incrementing.
+ if (failure) {
+ cbte->blocks_failed++;
+ }
+
+ cbte->blocks_done++;
+
+ if (node) {
+ toku_free(node);
+ }
+
+ // Print the status of this block to the console.
+ report(cbte->blocks_done, cbte->blocks_failed, cbte->total_blocks);
+ // We need to ALWAYS return 0 if we want to continue iterating
+ // through the nodes in the file.
+ r = 0;
+ return r;
+}
+
+// This calls toku_blocktable_iterate on the given block table.
+// Passes our check_block() function to be called as we iterate over
+// the block table. This will print any interesting failures and
+// update us on our progress.
+static void check_block_table(int fd, block_table *bt, struct ft *h) {
+ int64_t num_blocks = bt->get_blocks_in_use_unlocked();
+ printf("Starting verification of checkpoint containing");
+ printf(" %" PRId64 " blocks.\n", num_blocks);
+ fflush(stdout);
+
+ struct check_block_table_extra extra = { .fd = fd,
+ .blocks_done = 0,
+ .blocks_failed = 0,
+ .total_blocks = num_blocks,
+ .h = h };
+ int r = bt->iterate(block_table::TRANSLATION_CURRENT,
+ check_block,
+ &extra,
+ true,
+ true);
+ if (r != 0) {
+ // We can print more information here if necessary.
+ }
+
+ assert(extra.blocks_done == extra.total_blocks);
+ printf("Finished verification. ");
+ printf(" %" PRId64 " blocks checked,", extra.blocks_done);
+ printf(" %" PRId64 " bad block(s) detected\n", extra.blocks_failed);
+ fflush(stdout);
+}
+
+int
+main(int argc, char const * const argv[])
+{
+ // open the file
+ int r = 0;
+ int dictfd;
+ const char *dictfname, *outfname;
+ if (argc < 3 || argc > 4) {
+ fprintf(stderr, "%s: Invalid arguments.\n", argv[0]);
+ fprintf(stderr, "Usage: %s <dictionary> <logfile> [report%%]\n", argv[0]);
+ r = EX_USAGE;
+ goto exit;
+ }
+
+ assert(argc == 3 || argc == 4);
+ dictfname = argv[1];
+ outfname = argv[2];
+ if (argc == 4) {
+ set_errno(0);
+ pct = strtod(argv[3], NULL);
+ assert_zero(get_maybe_error_errno());
+ assert(pct > 0.0 && pct <= 100.0);
+ }
+
+ // Open the file as read-only.
+ dictfd = open(dictfname, O_RDONLY | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO);
+ if (dictfd < 0) {
+ perror(dictfname);
+ fflush(stderr);
+ abort();
+ }
+ outf = fopen(outfname, "w");
+ if (!outf) {
+ perror(outfname);
+ fflush(stderr);
+ abort();
+ }
+
+ // body of toku_ft_serialize_init();
+ num_cores = toku_os_get_number_active_processors();
+ r = toku_thread_pool_create(&ft_pool, num_cores); lazy_assert_zero(r);
+ assert_zero(r);
+
+ // deserialize the header(s)
+ struct ft *h1, *h2;
+ deserialize_headers(dictfd, &h1, &h2);
+
+ // walk over the block table and check blocks
+ if (h1) {
+ printf("Checking dictionary from header 1.\n");
+ check_block_table(dictfd, &h1->blocktable, h1);
+ }
+ if (h2) {
+ printf("Checking dictionary from header 2.\n");
+ check_block_table(dictfd, &h2->blocktable, h2);
+ }
+ if (h1 == NULL && h2 == NULL) {
+ printf("Both headers have a corruption and could not be used.\n");
+ }
+
+ toku_thread_pool_destroy(&ft_pool);
+exit:
+ return r;
+}
diff --git a/storage/tokudb/PerconaFT/tools/pmprof b/storage/tokudb/PerconaFT/tools/pmprof
new file mode 100644
index 00000000..de0a7e3c
--- /dev/null
+++ b/storage/tokudb/PerconaFT/tools/pmprof
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# a poor man's profiler
+# http://webcache.googleusercontent.com/search?q=cache:http://mituzas.lt/2009/02/15/poor-mans-contention-profiling/
+
+nsamples=1
+sleeptime=1
+
+while [ $# -gt 0 ] ; do
+ arg=$1;
+ if [[ $arg =~ --(.*)=(.*) ]] ; then
+ eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
+ else
+ break
+ fi
+ shift
+done
+
+pid=$1
+
+for x in $(seq 1 $nsamples)
+ do
+ gdb -ex "set pagination 0" -ex "thread apply all bt" -batch -p $pid
+ sleep $sleeptime
+ done | \
+awk '
+ BEGIN { s = ""; }
+ /^Thread/ { if (s != "") print s; s = ""; }
+ /^\#/ { if ($3 == "in") { v = $4; } else { v = $2 } if (s != "" ) { s = s "," v} else { s = v } }
+ END { print s }' | \
+sort | uniq -c | sort -r -n -k 1,1
diff --git a/storage/tokudb/PerconaFT/tools/tdb-recover.cc b/storage/tokudb/PerconaFT/tools/tdb-recover.cc
new file mode 100644
index 00000000..f01d0109
--- /dev/null
+++ b/storage/tokudb/PerconaFT/tools/tdb-recover.cc
@@ -0,0 +1,80 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+/* Recover an env. The logs are in argv[1]. The new database is created in the cwd. */
+
+// Test:
+// cd ../src/tests/tmpdir
+// ../../../ft/recover ../dir.test_log2.c.tdb
+
+#include "ft/ft-ops.h"
+#include "ft/logger/recover.h"
+
+static int recovery_main(int argc, const char *const argv[]);
+
+int main(int argc, const char *const argv[]) {
+ int r = toku_ft_layer_init();
+ assert(r == 0);
+ r = recovery_main(argc, argv);
+ toku_ft_layer_destroy();
+ return r;
+}
+
+int recovery_main (int argc, const char *const argv[]) {
+ const char *data_dir, *log_dir;
+ if (argc==3) {
+ data_dir = argv[1];
+ log_dir = argv[2];
+ } else if (argc==2) {
+ data_dir = log_dir = argv[1];
+ } else {
+ printf("Usage: %s <datadir> [ <logdir> ]\n", argv[0]);
+ return(1);
+ }
+
+ int r = tokuft_recover(nullptr,
+ nullptr,
+ nullptr,
+ nullptr,
+ data_dir, log_dir, nullptr, nullptr, nullptr, nullptr, 0);
+ if (r!=0) {
+ fprintf(stderr, "Recovery failed\n");
+ return(1);
+ }
+ return 0;
+}
diff --git a/storage/tokudb/PerconaFT/tools/tokudb_dump.cc b/storage/tokudb/PerconaFT/tools/tokudb_dump.cc
new file mode 100644
index 00000000..d7362fc6
--- /dev/null
+++ b/storage/tokudb/PerconaFT/tools/tokudb_dump.cc
@@ -0,0 +1,685 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+#include <db.h>
+
+#include <toku_stdlib.h>
+#include <toku_stdint.h>
+#include <toku_portability.h>
+#include <toku_assert.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+#include <memory.h>
+
+typedef struct {
+ bool leadingspace;
+ bool plaintext;
+ bool header;
+ bool footer;
+ bool is_private;
+ bool recovery_and_txn;
+ char* progname;
+ char* homedir;
+ char* database;
+ char* subdatabase;
+ int exitcode;
+ int recover_flags;
+ DBTYPE dbtype;
+ DBTYPE opened_dbtype;
+ DB* db;
+ DB_ENV* dbenv;
+} dump_globals;
+
+dump_globals g;
+
+#define SET_BITS(bitvector, bits) ((bitvector) |= (bits))
+#define REMOVE_BITS(bitvector, bits) ((bitvector) &= ~(bits))
+#define IS_SET_ANY(bitvector, bits) ((bitvector) & (bits))
+#define IS_SET_ALL(bitvector, bits) (((bitvector) & (bits)) == (bits))
+
+#define IS_POWER_OF_2(num) ((num) > 0 && ((num) & ((num) - 1)) == 0)
+
+//DB_ENV->err disabled since it does not use db_strerror
+#define PRINT_ERROR(retval, ...) \
+do { \
+if (0) g.dbenv->err(g.dbenv, retval, __VA_ARGS__); \
+else { \
+ fprintf(stderr, "\tIn %s:%d %s()\n", __FILE__, __LINE__, __FUNCTION__); \
+ fprintf(stderr, "%s: %s:", g.progname, db_strerror(retval)); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, "\n"); \
+ fflush(stderr); \
+} \
+} while (0)
+
+//DB_ENV->err disabled since it does not use db_strerror, errx does not exist.
+#define PRINT_ERRORX(...) \
+do { \
+if (0) g.dbenv->err(g.dbenv, 0, __VA_ARGS__); \
+else { \
+ fprintf(stderr, "\tIn %s:%d %s()\n", __FILE__, __LINE__, __FUNCTION__); \
+ fprintf(stderr, "%s: ", g.progname); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, "\n"); \
+ fflush(stderr); \
+} \
+} while (0)
+
+int strtoint32 (char* str, int32_t* num, int32_t min, int32_t max, int base);
+int strtouint32 (char* str, uint32_t* num, uint32_t min, uint32_t max, int base);
+int strtoint64 (char* str, int64_t* num, int64_t min, int64_t max, int base);
+int strtouint64 (char* str, uint64_t* num, uint64_t min, uint64_t max, int base);
+
+/*
+ * Convert a string to an integer of type "type".
+ *
+ *
+ * Sets errno and returns:
+ * EINVAL: str == NULL, num == NULL, or string not of the form [ \t]*[+-]?[0-9]+
+ * ERANGE: value out of range specified. (Range of [min, max])
+ *
+ * *num is unchanged on error.
+ * Returns:
+ *
+ */
+#define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \
+int name(char* str, type* num, type min, type max, int base) \
+{ \
+ char* test; \
+ bigtype value; \
+ \
+ assert(str); \
+ assert(num); \
+ assert(min <= max); \
+ assert(g.dbenv || g.progname); \
+ assert(base == 0 || (base >= 2 && base <= 36)); \
+ \
+ errno = 0; \
+ while (isspace(*str)) str++; \
+ value = strtofunc(str, &test, base); \
+ if ((*test != '\0' && *test != '\n') || test == str) { \
+ PRINT_ERRORX("%s: Invalid numeric argument\n", str); \
+ errno = EINVAL; \
+ goto error; \
+ } \
+ if (errno != 0) { \
+ PRINT_ERROR(errno, "%s\n", str); \
+ } \
+ if (value < min) { \
+ PRINT_ERRORX("%s: Less than minimum value (%" frmt ")\n", str, min); \
+ goto error; \
+ } \
+ if (value > max) { \
+ PRINT_ERRORX("%s: Greater than maximum value (%" frmt ")\n", str, max); \
+ goto error; \
+ } \
+ *num = value; \
+ return EXIT_SUCCESS; \
+error: \
+ return errno; \
+}
+
+DEF_STR_TO(strtoint32, int32_t, int64_t, strtoll, PRId32)
+DEF_STR_TO(strtouint32, uint32_t, uint64_t, strtoull, PRIu32)
+DEF_STR_TO(strtoint64, int64_t, int64_t, strtoll, PRId64)
+DEF_STR_TO(strtouint64, uint64_t, uint64_t, strtoull, PRIu64)
+
+static inline void
+outputbyte(uint8_t ch)
+{
+ if (g.plaintext) {
+ if (ch == '\\') printf("\\\\");
+ else if (isprint(ch)) printf("%c", ch);
+ else printf("\\%02x", ch);
+ }
+ else printf("%02x", ch);
+}
+
+static inline void
+outputstring(char* str)
+{
+ char* p;
+
+ for (p = str; *p != '\0'; p++) {
+ outputbyte((uint8_t)*p);
+ }
+}
+
+static inline void
+outputplaintextstring(char* str)
+{
+ bool old_plaintext = g.plaintext;
+ g.plaintext = true;
+ outputstring(str);
+ g.plaintext = old_plaintext;
+}
+
+static inline int
+verify_library_version(void)
+{
+ int major;
+ int minor;
+
+ db_version(&major, &minor, NULL);
+ if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR) {
+ PRINT_ERRORX("version %d.%d doesn't match library version %d.%d\n",
+ DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor);
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}
+
+static int last_caught = 0;
+
+static void catch_signal(int which_signal) {
+ last_caught = which_signal;
+ if (last_caught == 0) last_caught = SIGINT;
+}
+
+static inline void
+init_catch_signals(void) {
+ signal(SIGINT, catch_signal);
+ signal(SIGTERM, catch_signal);
+#ifdef SIGHUP
+ signal(SIGHUP, catch_signal);
+#endif
+#ifdef SIGPIPE
+ signal(SIGPIPE, catch_signal);
+#endif
+}
+
+static inline int
+caught_any_signals(void) {
+ return last_caught != 0;
+}
+
+static inline void
+resend_signals(void) {
+ if (last_caught) {
+ signal(last_caught, SIG_DFL);
+ raise(last_caught);
+ }
+}
+
+static int usage (void);
+static int create_init_env(void);
+static int dump_database (void);
+static int open_database (void);
+static int dump_pairs (void);
+static int dump_footer (void);
+static int dump_header (void);
+static int close_database (void);
+
+int main(int argc, char *const argv[]) {
+ int ch;
+ int retval;
+
+ /* Set up the globals. */
+ memset(&g, 0, sizeof(g));
+ g.leadingspace = true;
+ //TODO: Uncomment when DB_UNKNOWN + db->get_type are implemented.
+ g.dbtype = DB_UNKNOWN;
+ //g.dbtype = DB_BTREE;
+ g.progname = argv[0];
+ g.header = true;
+ g.footer = true;
+ g.recovery_and_txn = true;
+
+ if (verify_library_version() != 0) goto error;
+
+ while ((ch = getopt(argc, argv, "d:f:h:klNP:ps:RrVTx")) != EOF) {
+ switch (ch) {
+ case ('d'): {
+ PRINT_ERRORX("-%c option not supported.\n", ch);
+ goto error;
+ }
+ case ('f'): {
+ if (freopen(optarg, "w", stdout) == NULL) {
+ fprintf(stderr,
+ "%s: %s: reopen: %s\n",
+ g.progname, optarg, strerror(errno));
+ goto error;
+ }
+ break;
+ }
+ case ('h'): {
+ g.homedir = optarg;
+ break;
+ }
+ case ('k'): {
+ PRINT_ERRORX("-%c option not supported.\n", ch);
+ goto error;
+ }
+ case ('l'): {
+ //TODO: Implement (Requires master database support)
+ PRINT_ERRORX("-%c option not supported.\n", ch); //YET!
+ goto error;
+ }
+ case ('N'): {
+ PRINT_ERRORX("-%c option not supported.\n", ch);
+ goto error;
+ }
+ case ('P'): {
+ /* Clear password. */
+ memset(optarg, 0, strlen(optarg));
+ PRINT_ERRORX("-%c option not supported.\n", ch);
+ goto error;
+ }
+ case ('p'): {
+ g.plaintext = true;
+ break;
+ }
+ case ('R'): {
+ //TODO: Uncomment when DB_SALVAGE,DB_AGGRESSIVE are implemented.
+ /*g.recover_flags |= DB_SALVAGE | DB_AGGRESSIVE;*/
+
+ //TODO: Implement aggressive recovery (requires db->verify())
+ PRINT_ERRORX("-%c option not supported.\n", ch);
+ goto error;
+ }
+ case ('r'): {
+ //TODO: Uncomment when DB_SALVAGE,DB_AGGRESSIVE are implemented.
+ /*g.recover_flags |= DB_SALVAGE;*/
+
+ //TODO: Implement recovery (requires db->verify())
+ PRINT_ERRORX("-%c option not supported.\n", ch);
+ goto error;
+ }
+ case ('s'): {
+ g.subdatabase = optarg;
+ break;
+ }
+ case ('V'): {
+ printf("%s\n", db_version(NULL, NULL, NULL));
+ goto cleanup;
+ }
+ case ('T'): {
+ g.plaintext = true;
+ g.leadingspace = false;
+ g.header = false;
+ g.footer = false;
+ break;
+ }
+ case ('x'): {
+ g.recovery_and_txn = false;
+ break;
+ }
+ case ('?'):
+ default: {
+ g.exitcode = usage();
+ goto cleanup;
+ }
+ }
+ }
+ argc -= optind;
+ argv += optind;
+
+ //TODO: Uncomment when DB_SALVAGE,DB_AGGRESSIVE,DB_PRINTABLE,db->verify are implemented.
+ /*
+ if (g.plaintext) g.recover_flags |= DB_PRINTABLE;
+
+ if (g.subdatabase != NULL && IS_SET_ALL(g.recover_flags, DB_SALVAGE)) {
+ if (IS_SET_ALL(g.recover_flags, DB_AGGRESSIVE)) {
+ PRINT_ERRORX("The -s and -R options may not both be specified.\n");
+ goto error;
+ }
+ PRINT_ERRORX("The -s and -r options may not both be specified.\n");
+ goto error;
+
+ }
+ */
+
+ if (argc != 1) {
+ g.exitcode = usage();
+ goto cleanup;
+ }
+
+ init_catch_signals();
+
+ g.database = argv[0];
+ if (caught_any_signals()) goto cleanup;
+ if (create_init_env() != 0) goto error;
+ if (caught_any_signals()) goto cleanup;
+ if (dump_database() != 0) goto error;
+ if (false) {
+error:
+ g.exitcode = EXIT_FAILURE;
+ fprintf(stderr, "%s: Quitting out due to errors.\n", g.progname);
+ }
+cleanup:
+ if (g.dbenv && (retval = g.dbenv->close(g.dbenv, 0)) != 0) {
+ g.exitcode = EXIT_FAILURE;
+ fprintf(stderr, "%s: %s: dbenv->close\n", g.progname, db_strerror(retval));
+ }
+ // if (g.subdatabase) free(g.subdatabase);
+ resend_signals();
+
+ return g.exitcode;
+}
+
+int dump_database()
+{
+ int retval;
+
+ /* Create a database handle. */
+ retval = db_create(&g.db, g.dbenv, 0);
+ if (retval != 0) {
+ PRINT_ERROR(retval, "db_create");
+ return EXIT_FAILURE;
+ }
+
+ /*
+ TODO: If/when supporting encryption
+ if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) {
+ PRINT_ERROR(ret, "DB->set_flags: DB_ENCRYPT");
+ goto error;
+ }
+ */
+ if (open_database() != 0) goto error;
+ if (caught_any_signals()) goto cleanup;
+ if (g.header && dump_header() != 0) goto error;
+ if (caught_any_signals()) goto cleanup;
+ if (dump_pairs() != 0) goto error;
+ if (caught_any_signals()) goto cleanup;
+ if (g.footer && dump_footer() != 0) goto error;
+
+ if (false) {
+error:
+ g.exitcode = EXIT_FAILURE;
+ }
+cleanup:
+
+ if (close_database() != 0) g.exitcode = EXIT_FAILURE;
+
+ return g.exitcode;
+}
+
+int usage()
+{
+ fprintf(stderr,
+ "usage: %s [-pVT] [-x] [-f output] [-h home] [-s database] db_file\n",
+ g.progname);
+ return EXIT_FAILURE;
+}
+
+int create_init_env()
+{
+ int retval;
+ DB_ENV* dbenv;
+ int flags;
+ //TODO: Experiments to determine right cache size for tokudb, or maybe command line argument.
+
+ retval = db_env_create(&dbenv, 0);
+ if (retval) {
+ fprintf(stderr, "%s: db_dbenv_create: %s\n", g.progname, db_strerror(retval));
+ goto error;
+ }
+ ///TODO: UNCOMMENT/IMPLEMENT dbenv->set_errfile(dbenv, stderr);
+ dbenv->set_errpfx(dbenv, g.progname);
+ /*
+ TODO: Anything for encryption?
+ */
+
+ /* Open the dbenvironment. */
+ g.is_private = false;
+ //flags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_USE_ENVIRON;
+ flags = DB_INIT_LOCK | DB_INIT_MPOOL; ///TODO: UNCOMMENT/IMPLEMENT | DB_USE_ENVIRON;
+ if (g.recovery_and_txn) {
+ SET_BITS(flags, DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER);
+ }
+
+ /*
+ ///TODO: UNCOMMENT/IMPLEMENT Notes: We require DB_PRIVATE
+ if (!dbenv->open(dbenv, g.homedir, flags, 0)) goto success;
+ */
+
+ /*
+ ///TODO: UNCOMMENT/IMPLEMENT
+ retval = dbenv->set_cachesize(dbenv, 0, cache, 1);
+ if (retval) {
+ PRINT_ERROR(retval, "DB_ENV->set_cachesize");
+ goto error;
+ }
+ */
+ g.is_private = true;
+ //TODO: Do we want to support transactions even in single-process mode?
+ //Logging is not necessary.. this is read-only.
+ //However, do we need to use DB_INIT_LOG to join a logging environment?
+ //REMOVE_BITS(flags, DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN);
+ SET_BITS(flags, DB_CREATE | DB_PRIVATE);
+
+ retval = dbenv->open(dbenv, g.homedir, flags, 0);
+ if (retval) {
+ PRINT_ERROR(retval, "DB_ENV->open");
+ goto error;
+ }
+ g.dbenv = dbenv;
+ return EXIT_SUCCESS;
+
+error:
+ return EXIT_FAILURE;
+}
+
+#define DUMP_FLAG(bit, dump) if (IS_SET_ALL(flags, bit)) printf(dump);
+
+#define DUMP_IGNORED_FLAG(bit, dump)
+
+
+int dump_header()
+{
+ uint32_t flags;
+ int retval;
+ DB* db = g.db;
+
+ assert(g.header);
+ printf("VERSION=3\n");
+ printf("format=%s\n", g.plaintext ? "print" : "bytevalue");
+ //TODO: Uncomment when DB_UNKNOWN + db->get_type are implemented.
+ /*assert(g.dbtype == DB_BTREE || (g.dbtype == DB_UNKNOWN && g.opened_dbtype == DB_BTREE));*/
+ printf("type=btree\n");
+ //TODO: Get page size from db. Currently tokudb does not support db->get_pagesize.
+ //Don't print this out //printf("db_pagesize=4096\n");
+ if (g.subdatabase) {
+ printf("subdatabase=");
+ outputplaintextstring(g.subdatabase);
+ printf("\n");
+ }
+ //TODO: Uncomment when db->get_flags is implemented
+ if ((retval = db->get_flags(db, &flags)) != 0) {
+ PRINT_ERROR(retval, "DB->get_flags");
+ goto error;
+ }
+ DUMP_IGNORED_FLAG(DB_CHKSUM, "chksum=1\n");
+ DUMP_IGNORED_FLAG(DB_RECNUM, "recnum=1\n");
+ printf("HEADER=END\n");
+
+ if (ferror(stdout)) goto error;
+ return EXIT_SUCCESS;
+
+error:
+ return EXIT_FAILURE;
+}
+
+int dump_footer()
+{
+ printf("DATA=END\n");
+ if (ferror(stdout)) goto error;
+
+ return EXIT_SUCCESS;
+error:
+ return EXIT_FAILURE;
+}
+
+int open_database()
+{
+ DB* db = g.db;
+ int retval;
+
+ int open_flags = 0;//|DB_RDONLY;
+ //TODO: Transaction auto commit stuff
+ SET_BITS(open_flags, DB_AUTO_COMMIT);
+
+ retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666);
+ if (retval != 0) {
+ PRINT_ERROR(retval, "DB->open: %s", g.database);
+ goto error;
+ }
+ //TODO: Uncomment when DB_UNKNOWN + db->get_type are implemented.
+ /*
+ retval = db->get_type(db, &g.opened_dbtype);
+ if (retval != 0) {
+ PRINT_ERROR(retval, "DB->get_type");
+ goto error;
+ }
+ if (g.opened_dbtype != DB_BTREE) {
+ PRINT_ERRORX("Unsupported db type %d\n", g.opened_dbtype);
+ goto error;
+ }
+ if (g.dbtype != DB_UNKNOWN && g.opened_dbtype != g.dbtype) {
+ PRINT_ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, g.opened_dbtype);
+ goto error;
+ }*/
+ return EXIT_SUCCESS;
+error:
+ fprintf(stderr, "Quitting out due to errors.\n");
+ return EXIT_FAILURE;
+}
+
+static int dump_dbt(DBT* dbt)
+{
+ char* str;
+ uint32_t idx;
+
+ assert(dbt);
+ str = (char*)dbt->data;
+ if (g.leadingspace) printf(" ");
+ if (dbt->size > 0) {
+ assert(dbt->data);
+ for (idx = 0; idx < dbt->size; idx++) {
+ outputbyte(str[idx]);
+ if (ferror(stdout)) {
+ perror("stdout");
+ goto error;
+ }
+ }
+ }
+ printf("\n");
+ if (false) {
+error:
+ g.exitcode = EXIT_FAILURE;
+ }
+ return g.exitcode;
+}
+
+int dump_pairs()
+{
+ int retval;
+ DBT key;
+ DBT data;
+ DB* db = g.db;
+ DBC* dbc = NULL;
+
+ memset(&key, 0, sizeof(key));
+ memset(&data, 0, sizeof(data));
+
+ DB_TXN* txn = NULL;
+ if (g.recovery_and_txn) {
+ retval = g.dbenv->txn_begin(g.dbenv, NULL, &txn, 0);
+ if (retval) {
+ PRINT_ERROR(retval, "DB_ENV->txn_begin");
+ goto error;
+ }
+ }
+
+ if ((retval = db->cursor(db, txn, &dbc, 0)) != 0) {
+ PRINT_ERROR(retval, "DB->cursor");
+ goto error;
+ }
+ while ((retval = dbc->c_get(dbc, &key, &data, DB_NEXT)) == 0) {
+ if (caught_any_signals()) goto cleanup;
+ if (dump_dbt(&key) != 0) goto error;
+ if (dump_dbt(&data) != 0) goto error;
+ }
+ if (retval != DB_NOTFOUND) {
+ PRINT_ERROR(retval, "DBC->c_get");
+ goto error;
+ }
+
+
+ if (false) {
+error:
+ g.exitcode = EXIT_FAILURE;
+ }
+cleanup:
+ if (dbc && (retval = dbc->c_close(dbc)) != 0) {
+ PRINT_ERROR(retval, "DBC->c_close");
+ g.exitcode = EXIT_FAILURE;
+ }
+ if (txn) {
+ if (retval) {
+ int r2 = txn->abort(txn);
+ if (r2) PRINT_ERROR(r2, "DB_TXN->abort");
+ }
+ else {
+ retval = txn->commit(txn, 0);
+ if (retval) PRINT_ERROR(retval, "DB_TXN->abort");
+ }
+ }
+ return g.exitcode;
+}
+
+int close_database()
+{
+ DB* db = g.db;
+ int retval;
+
+ assert(db);
+ if ((retval = db->close(db, 0)) != 0) {
+ PRINT_ERROR(retval, "DB->close");
+ goto error;
+ }
+ return EXIT_SUCCESS;
+error:
+ return EXIT_FAILURE;
+}
diff --git a/storage/tokudb/PerconaFT/tools/tokuft_logprint.cc b/storage/tokudb/PerconaFT/tools/tokuft_logprint.cc
new file mode 100644
index 00000000..924eee2d
--- /dev/null
+++ b/storage/tokudb/PerconaFT/tools/tokuft_logprint.cc
@@ -0,0 +1,74 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+/* Dump the log from stdin to stdout. */
+#include "ft/ft.h"
+#include "ft/log_header.h"
+#include "ft/logger/logger.h"
+
+using namespace std;
+
+int main (int argc, const char *const argv[]) {
+ int r = toku_ft_layer_init();
+ assert_zero(r);
+
+ int count=-1;
+ while (argc>1) {
+ if (strcmp(argv[1], "--oldcode")==0) {
+ fprintf(stderr,"Old code no longer works.\n");
+ exit(1);
+ } else {
+ count = atoi(argv[1]);
+ }
+ argc--; argv++;
+ }
+ int i;
+ uint32_t version;
+ r = toku_read_and_print_logmagic(stdin, &version);
+ for (i=0; i!=count; i++) {
+ r = toku_logprint_one_record(stdout, stdin);
+ if (r==EOF) break;
+ if (r!=0) {
+ fflush(stdout);
+ fprintf(stderr, "Problem in log err=%d\n", r);
+ exit(1);
+ }
+ }
+ toku_ft_layer_destroy();
+ return 0;
+}
diff --git a/storage/tokudb/PerconaFT/tools/tokuftdump.cc b/storage/tokudb/PerconaFT/tools/tokuftdump.cc
new file mode 100644
index 00000000..44edb151
--- /dev/null
+++ b/storage/tokudb/PerconaFT/tools/tokuftdump.cc
@@ -0,0 +1,1246 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+// Dump a fractal tree file
+
+#include <ctype.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <map>
+#include <string>
+#include <string.h>
+#include "ft/serialize/block_table.h"
+#include "ft/cachetable/cachetable.h"
+#include "ft/ft.h"
+#include "ft/ft-internal.h"
+#include "ft/serialize/ft-serialize.h"
+#include "ft/serialize/ft_node-serialize.h"
+#include "ft/node.h"
+
+using namespace std;
+
+static int do_dump_data = 1;
+static int do_interactive = 0;
+static int do_json = 0;
+static int do_header = 0;
+static int do_fragmentation = 0;
+static int do_garbage = 0;
+static int do_translation_table = 0;
+static int do_summary = 0;
+static int do_rootnode = 0;
+static int do_node = 0;
+static BLOCKNUM do_node_num;
+static int do_tsv = 0;
+static const char *arg0;
+static const char *fname;
+
+//it holdes the messges count for each FT's node
+typedef struct nodeMessage{
+ int id;
+ int clean;//0=clean >=1 dirty
+ int *count;//holds the messages
+ nodeMessage *nextNode;
+}NMC;
+enum { maxline = 128};
+
+static int printNodeMessagesToSTDout(NMC* ptr);
+
+static int printLevelSTDout(int *);
+
+static void treeToSTDout(NMC *msgs[], int height);
+
+static void format_time(const uint64_t time_int, char *buf) {
+ time_t timer = (time_t) time_int;
+ ctime_r(&timer, buf);
+ assert(buf[24] == '\n');
+ buf[24] = 0;
+}
+
+static void print_item(const void *val, uint32_t len) {
+ printf("\"");
+ uint32_t i;
+ for (i=0; i<len; i++) {
+ unsigned char ch = ((unsigned char*)val)[i];
+ if (isprint(ch) && ch!='\\' && ch!='"') {
+ printf("%c", ch);
+ } else {
+ printf("\\%03o", ch);
+ }
+ }
+ printf("\"");
+}
+
+static void simple_hex_dump(unsigned char *vp, uint64_t size) {
+ for (uint64_t i = 0; i < size; i++) {
+ unsigned char c = vp[i];
+ printf("%2.2X", c);
+ }
+}
+
+static void hex_dump(unsigned char *vp, uint64_t offset, uint64_t size) {
+ uint64_t n = size / 32;
+ for (uint64_t i = 0; i < n; i++) {
+ printf("%" PRIu64 ": ", offset);
+ for (uint64_t j = 0; j < 32; j++) {
+ unsigned char c = vp[j];
+ printf("%2.2X", c);
+ if (((j+1) % 4) == 0)
+ printf(" ");
+ }
+ for (uint64_t j = 0; j < 32; j++) {
+ unsigned char c = vp[j];
+ printf("%c", isprint(c) ? c : ' ');
+ }
+ printf("\n");
+ vp += 32;
+ offset += 32;
+ }
+ size = size % 32;
+ for (uint64_t i=0; i<size; i++) {
+ if ((i % 32) == 0)
+ printf("%" PRIu64 ": ", offset+i);
+ printf("%2.2X", vp[i]);
+ if (((i+1) % 4) == 0)
+ printf(" ");
+ if (((i+1) % 32) == 0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+static void dump_descriptor(DESCRIPTOR d) {
+ printf(" descriptor size %u ", d->dbt.size);
+ simple_hex_dump((unsigned char*) d->dbt.data, d->dbt.size);
+ printf("\n");
+}
+
+static void open_header(int fd, FT *header, CACHEFILE cf) {
+ FT ft = NULL;
+ int r;
+ const char *fn = toku_cachefile_fname_in_env(cf);
+ r = toku_deserialize_ft_from (fd, fn, MAX_LSN, &ft);
+ if (r != 0) {
+ fprintf(stderr, "%s: can not deserialize from %s error %d\n", arg0, fname, r);
+ exit(1);
+ }
+ assert_zero(r);
+ ft->cf = cf;
+ *header = ft;
+}
+
+static void dump_header(FT ft) {
+ char timestr[26];
+ printf("ft:\n");
+ printf(" layout_version=%d\n", ft->h->layout_version);
+ printf(" layout_version_original=%d\n", ft->h->layout_version_original);
+ printf(" layout_version_read_from_disk=%d\n", ft->layout_version_read_from_disk);
+ printf(" build_id=%d\n", ft->h->build_id);
+ printf(" build_id_original=%d\n", ft->h->build_id_original);
+ format_time(ft->h->time_of_creation, timestr);
+ printf(" time_of_creation= %" PRIu64 " %s\n", ft->h->time_of_creation, timestr);
+ format_time(ft->h->time_of_last_modification, timestr);
+ printf(" time_of_last_modification=%" PRIu64 " %s\n", ft->h->time_of_last_modification, timestr);
+ printf(" dirty=%d\n", ft->h->dirty());
+ printf(" checkpoint_count=%" PRId64 "\n", ft->h->checkpoint_count);
+ printf(" checkpoint_lsn=%" PRId64 "\n", ft->h->checkpoint_lsn.lsn);
+ printf(" nodesize=%u\n", ft->h->nodesize);
+ printf(" fanout=%u\n", ft->h->fanout);
+ printf(" basementnodesize=%u\n", ft->h->basementnodesize);
+ printf(" compression_method=%u\n", (unsigned) ft->h->compression_method);
+ printf(" unnamed_root=%" PRId64 "\n", ft->h->root_blocknum.b);
+ printf(" flags=%u\n", ft->h->flags);
+ dump_descriptor(&ft->descriptor);
+ printf(" estimated numrows=%" PRId64 "\n", ft->in_memory_stats.numrows);
+ printf(" estimated numbytes=%" PRId64 "\n", ft->in_memory_stats.numbytes);
+ printf(" logical row count=%" PRId64 "\n", ft->in_memory_logical_rows);
+}
+
+static int64_t getRootNode(FT ft) {
+ return ft->h->root_blocknum.b;
+}
+
+static int print_le(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t idx UU(), void *const ai UU()) {
+ unsigned int *le_index = (unsigned int *) ai;
+ printf("%u: ", *le_index); *le_index += 1;
+ print_klpair(stdout, key, keylen, le);
+ printf("\n");
+ return 0;
+}
+
+static int getHeight(int fd, BLOCKNUM blocknum, FT ft){
+ FTNODE n;
+ FTNODE_DISK_DATA ndd = nullptr;
+ ftnode_fetch_extra bfe;
+ bfe.create_for_full_read(ft);
+ int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe);
+ assert_zero(r);
+ assert(n!=0);
+ return n->height;
+}
+
+static FTNODE getNode(int fd, BLOCKNUM blocknum, FT ft) {
+ FTNODE n;
+ FTNODE_DISK_DATA ndd = nullptr;
+ ftnode_fetch_extra bfe;
+ bfe.create_for_full_read(ft);
+ int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe);
+ assert_zero(r);;
+ return n;
+}
+
+static int countNodes(NMC *level){
+ int count=0;
+ NMC *ptr=level;
+ while(ptr!=NULL){
+ count++;
+ ptr=ptr->nextNode;
+ }
+ return count;
+}
+
+static int * countMessages(NMC *level){
+ int *counts=new int[16];
+ for(int i=0;i<16;i++){
+ counts[i]=0;
+ }
+ NMC *ptr=level;
+ while(ptr!=NULL){
+ for(int i=0;i<16;i++){
+ counts[i]+=ptr->count[i];
+ }
+ ptr=ptr->nextNode;
+ }
+ return counts;
+}
+
+static NMC * getLast(NMC *level){
+ if (level==NULL) return NULL;
+ NMC *ptr=level;
+ while(ptr->nextNode!=NULL){
+ ptr=ptr->nextNode;
+ }
+ return ptr;
+}
+
+/*
+ * Prints the total messages at each to STDout
+ */
+static int printLevelSTDout(int *count){
+ int isEmpty=0;
+ for(int j=0;j<16;j++){
+ if(count[j]>0){
+ cout <<count[j]<<" ";
+ isEmpty++;
+ switch (j) {
+ case FT_INSERT: cout <<"INSERT(s) "; break;
+ case FT_INSERT_NO_OVERWRITE: cout <<"INSERT_NO_OVERWRITE(s) "; break;
+ case FT_DELETE_ANY: cout <<"DELETE_ANY(s) "; break;
+ case FT_ABORT_ANY: cout <<"ABORT_ANY(s) "; break;
+ case FT_COMMIT_ANY: cout <<"COMMIT_ANY(s) "; break;
+ case FT_COMMIT_BROADCAST_ALL: cout <<"COMMIT_BROADCAST_ALL(s) "; break;
+ case FT_COMMIT_BROADCAST_TXN: cout <<"COMMIT_BROADCAST_TXN(s) "; break;
+ case FT_ABORT_BROADCAST_TXN: cout <<"ABORT_BROADCAST_TXN(s) "; break;
+ case FT_OPTIMIZE: cout <<"OPTIMIZE(s) "; break;
+ case FT_OPTIMIZE_FOR_UPGRADE: cout <<"OPTIMIZE_FOR_UPGRADE(s) "; break;
+ case FT_UPDATE: cout <<"UPDATE(s) "; break;
+ case FT_UPDATE_BROADCAST_ALL: cout <<"UPDATE_BROADCAST_ALL(s) "; break;
+ }
+
+ }
+ }
+ return isEmpty;
+}
+
+/*
+ * Prints the total # of messages in a node to STD output
+ */
+static int printNodeMessagesToSTDout(NMC *ptr){
+ cout <<"\nNode :"<<ptr->id<<" has :";
+ for(int j=0;j<16;j++){
+ if(ptr->count[j]>0){
+ cout <<ptr->count[j]<<" ";
+ switch (j) {
+ case FT_INSERT: cout <<"INSERT(s) "; break;
+ case FT_INSERT_NO_OVERWRITE: cout <<"INSERT_NO_OVERWRITE(s) "; break;
+ case FT_DELETE_ANY: cout <<"DELETE_ANY(s) "; break;
+ case FT_ABORT_ANY: cout <<"ABORT_ANY(s) "; break;
+ case FT_COMMIT_ANY: cout <<"COMMIT_ANY(s) "; break;
+ case FT_COMMIT_BROADCAST_ALL: cout <<"COMMIT_BROADCAST_ALL(s) "; break;
+ case FT_COMMIT_BROADCAST_TXN: cout <<"COMMIT_BROADCAST_TXN(s) "; break;
+ case FT_ABORT_BROADCAST_TXN: cout <<"ABORT_BROADCAST_TXN(s) "; break;
+ case FT_OPTIMIZE: cout <<"OPTIMIZE(s) "; break;
+ case FT_OPTIMIZE_FOR_UPGRADE: cout <<"OPTIMIZE_FOR_UPGRADE(s) "; break;
+ case FT_UPDATE: cout <<"UPDATE(s) "; break;
+ case FT_UPDATE_BROADCAST_ALL: cout <<"UPDATE_BROADCAST_ALL(s) "; break;
+ }
+ }
+ }
+ return 1;
+}
+
+static void levelToSTDout(NMC *list, int level){
+ NMC *ptr=list;
+ cout <<endl<<"Height : "<<level<<endl;
+ while(ptr!=NULL){
+ if(ptr->clean!=0){
+ printNodeMessagesToSTDout(ptr);
+ }
+ else{
+ cout << "\nNode : "<<ptr->id<<" has no messages";
+ }
+ ptr=ptr->nextNode;
+ }
+ cout <<endl;
+}
+
+/*
+ * prints the tree total # of nodes and total # of messages at each height in :
+ * STDout in human readable format
+ */
+static void treeToSTDout(NMC *msgs[], int height){
+ for(int i=height; i>=0 ; i--){
+ cout <<"At height "<<i;
+ int *counts=countMessages(msgs[i]);
+ cout <<"\n Node Count: "<< countNodes(msgs[i])<<endl;
+ cout <<" Messages: ";
+ if(printLevelSTDout(counts)==0) cout <<"0\n";
+ else cout <<endl;
+ }
+}
+
+//traverse through the FT and report back the count of messages in every node
+static void countMessagesInFT(int fd, BLOCKNUM blocknum, FT ft,NMC *msgs[]){
+ FTNODE n=getNode(fd,blocknum,ft);
+
+ NMC *last=NULL;
+ if(msgs[n->height]==NULL){
+ last = msgs[n->height]=new NMC;
+ }else {
+ last=getLast(msgs[n->height]);
+ last->nextNode=new NMC;
+ last=last->nextNode;
+ }
+ last->id=blocknum.b;
+ last->count=new int[16];
+ for(int i=0;i<16;i++){
+ last->count[i]=0;
+ }
+ last->clean=0;
+ last->nextNode=NULL;
+
+ if (n->height==0){
+ toku_ftnode_free(&n);
+ return;
+ }
+ for(int i=0;i<n->n_children;i++){
+ NONLEAF_CHILDINFO bnc = BNC(n, i);
+ if (n->height==1 && n->bp[i].ptr.tag==BCT_NULL){
+ cout <<n->bp[i].ptr.tag;
+ }
+ auto dump_fn=[&](const ft_msg &msg, bool UU(is_fresh)) {
+ enum ft_msg_type type = (enum ft_msg_type) msg.type();
+ last->count[type]++;
+ last->clean=1;
+ return 0;
+ };
+
+ bnc->msg_buffer.iterate(dump_fn);
+
+ blocknum=make_blocknum(BP_BLOCKNUM(n, i).b);
+ countMessagesInFT(fd,blocknum,ft, msgs);
+ }
+
+ toku_ftnode_free(&n);
+}
+
+static void dump_node(int fd, BLOCKNUM blocknum, FT ft) {
+ FTNODE n;
+ FTNODE_DISK_DATA ndd = nullptr;
+ ftnode_fetch_extra bfe;
+ bfe.create_for_full_read(ft);
+ int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe);
+ assert_zero(r);
+ assert(n!=0);
+ printf("ftnode\n");
+ DISKOFF disksize, diskoffset;
+ ft->blocktable.translate_blocknum_to_offset_size(blocknum, &diskoffset, &disksize);
+ printf(" diskoffset =%" PRId64 "\n", diskoffset);
+ printf(" disksize =%" PRId64 "\n", disksize);
+ printf(" serialize_size =%u\n", toku_serialize_ftnode_size(n));
+ printf(" flags =%u\n", n->flags);
+ printf(" blocknum=%" PRId64 "\n", n->blocknum.b);
+ //printf(" log_lsn =%lld\n", n->log_lsn.lsn); // The log_lsn is a memory-only value.
+ printf(" height =%d\n", n->height);
+ printf(" layout_version=%d\n", n->layout_version);
+ printf(" layout_version_original=%d\n", n->layout_version_original);
+ printf(" layout_version_read_from_disk=%d\n", n->layout_version_read_from_disk);
+ printf(" build_id=%d\n", n->build_id);
+ printf(" max_msn_applied_to_node_on_disk=%" PRId64 " (0x%" PRIx64 ")\n", n->max_msn_applied_to_node_on_disk.msn, n->max_msn_applied_to_node_on_disk.msn);
+ printf(" io time %lf decompress time %lf deserialize time %lf\n",
+ tokutime_to_seconds(bfe.io_time),
+ tokutime_to_seconds(bfe.decompress_time),
+ tokutime_to_seconds(bfe.deserialize_time));
+
+ printf(" n_children=%d\n", n->n_children);
+ printf(" pivotkeys.total_size()=%u\n", (unsigned) n->pivotkeys.total_size());
+
+ if (n->height > 0) {
+ printf(" pivots:\n");
+ } else {
+ printf("LEAF keys:\n");
+ }
+
+ for (int i=0; i<n->n_children-1; i++) {
+ const DBT piv = n->pivotkeys.get_pivot(i);
+ printf(" pivot %2d:", i);
+ if (n->flags)
+ printf(" flags=%x ", n->flags);
+ print_item(piv.data, piv.size);
+ printf("\n");
+ }
+
+ if (n->height > 0) {
+ printf(" children:\n");
+ } else {
+ printf("LEAF data:\n");
+ }
+
+ for (int i=0; i<n->n_children; i++) {
+ printf(" child %d: ", i);
+ if (n->height > 0) {
+ printf("%" PRId64 "\n", BP_BLOCKNUM(n, i).b);
+ NONLEAF_CHILDINFO bnc = BNC(n, i);
+ unsigned int n_bytes = toku_bnc_nbytesinbuf(bnc);
+ int n_entries = toku_bnc_n_entries(bnc);
+ if (n_bytes > 0 || n_entries > 0) {
+ printf(" buffer contains %u bytes (%d items)\n", n_bytes, n_entries);
+ }
+ if (do_dump_data) {
+ struct dump_data_fn {
+ int operator()(const ft_msg &msg, bool UU(is_fresh)) {
+ enum ft_msg_type type = (enum ft_msg_type) msg.type();
+ MSN msn = msg.msn();
+ XIDS xids = msg.xids();
+ const void *key = msg.kdbt()->data;
+ const void *data = msg.vdbt()->data;
+ uint32_t keylen = msg.kdbt()->size;
+ uint32_t datalen = msg.vdbt()->size;
+ printf(" msn=%" PRIu64 " (0x%" PRIx64 ") ", msn.msn, msn.msn);
+ printf(" TYPE=");
+ switch (type) {
+ case FT_NONE: printf("NONE"); goto ok;
+ case FT_INSERT: printf("INSERT"); goto ok;
+ case FT_INSERT_NO_OVERWRITE: printf("INSERT_NO_OVERWRITE"); goto ok;
+ case FT_DELETE_ANY: printf("DELETE_ANY"); goto ok;
+ case FT_ABORT_ANY: printf("ABORT_ANY"); goto ok;
+ case FT_COMMIT_ANY: printf("COMMIT_ANY"); goto ok;
+ case FT_COMMIT_BROADCAST_ALL: printf("COMMIT_BROADCAST_ALL"); goto ok;
+ case FT_COMMIT_BROADCAST_TXN: printf("COMMIT_BROADCAST_TXN"); goto ok;
+ case FT_ABORT_BROADCAST_TXN: printf("ABORT_BROADCAST_TXN"); goto ok;
+ case FT_OPTIMIZE: printf("OPTIMIZE"); goto ok;
+ case FT_OPTIMIZE_FOR_UPGRADE: printf("OPTIMIZE_FOR_UPGRADE"); goto ok;
+ case FT_UPDATE: printf("UPDATE"); goto ok;
+ case FT_UPDATE_BROADCAST_ALL: printf("UPDATE_BROADCAST_ALL"); goto ok;
+ }
+ printf("HUH?");
+ok:
+ printf(" xid=");
+ toku_xids_fprintf(stdout, xids);
+ printf(" ");
+ print_item(key, keylen);
+ if (datalen>0) {
+ printf(" ");
+ print_item(data, datalen);
+ }
+ printf("\n");
+ return 0;
+ }
+ } dump_fn;
+ bnc->msg_buffer.iterate(dump_fn);
+ }
+ } else {
+ printf(" n_bytes_in_buffer= %" PRIu64 "", BLB_DATA(n, i)->get_disk_size());
+ printf(" items_in_buffer=%u\n", BLB_DATA(n, i)->num_klpairs());
+ if (do_dump_data) {
+ unsigned int le_index = 0;
+ BLB_DATA(n, i)->iterate<void, print_le>(&le_index);
+ }
+ }
+ }
+ toku_ftnode_free(&n);
+ toku_free(ndd);
+}
+
+static void dump_block_translation(FT ft, uint64_t offset) {
+ ft->blocktable.blocknum_dump_translation(make_blocknum(offset));
+}
+
+static void dump_fragmentation(int UU(f), FT ft, int tsv) {
+ int64_t used_space;
+ int64_t total_space;
+ ft->blocktable.internal_fragmentation(&total_space, &used_space);
+ int64_t fragsizes = total_space - used_space;
+
+ if (tsv) {
+ printf("%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%.1f\n", used_space, total_space, fragsizes,
+ 100. * ((double)fragsizes / (double)(total_space)));
+ } else {
+ printf("used_size\t%" PRId64 "\n", used_space);
+ printf("total_size\t%" PRId64 "\n", total_space);
+ printf("fragsizes\t%" PRId64 "\n", fragsizes);
+ printf("fragmentation\t%.1f\n", 100. * ((double)fragsizes / (double)(total_space)));
+ }
+}
+
+typedef struct {
+ int fd;
+ FT ft;
+ uint64_t blocksizes;
+ uint64_t leafsizes;
+ uint64_t leafblocks;
+} frag_help_extra;
+
+static int nodesizes_helper(BLOCKNUM b, int64_t size, int64_t UU(address), void *extra) {
+ frag_help_extra *CAST_FROM_VOIDP(info, extra);
+ FTNODE n;
+ FTNODE_DISK_DATA ndd = NULL;
+ ftnode_fetch_extra bfe;
+ bfe.create_for_full_read(info->ft);
+ int r = toku_deserialize_ftnode_from(info->fd, b, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe);
+ if (r==0) {
+ info->blocksizes += size;
+ if (n->height == 0) {
+ info->leafsizes += size;
+ info->leafblocks++;
+ }
+ toku_ftnode_free(&n);
+ toku_free(ndd);
+ }
+ return 0;
+}
+
+static void dump_nodesizes(int fd, FT ft) {
+ frag_help_extra info;
+ memset(&info, 0, sizeof(info));
+ info.fd = fd;
+ info.ft = ft;
+ ft->blocktable.iterate(block_table::TRANSLATION_CHECKPOINTED,
+ nodesizes_helper, &info, true, true);
+ printf("leafblocks\t%" PRIu64 "\n", info.leafblocks);
+ printf("blocksizes\t%" PRIu64 "\n", info.blocksizes);
+ printf("leafsizes\t%" PRIu64 "\n", info.leafsizes);
+}
+
+/* ===== struct and function to get a summary of atree ===== */
+
+typedef struct {
+ int fd;
+ FT ft;
+ uint64_t blocksizes;
+ uint64_t leafsizes;
+ uint64_t serialsize; // sizes of serialized data (assume uncomressed)
+ uint64_t leafblocks; // count of leaf nodes
+ uint64_t nonleafnode_cnt; // count of non-leaf nodes
+ uint64_t maxheight; // height of the tree
+ uint64_t msg_cnt; // message count in non-leafs
+ uint64_t msg_size; // size (in bytes of all messages in non-leafs
+ uint64_t pairs_cnt; // count of pairs in leaf nodes
+ std::map<int, int> height_cnt; // count of nodes per height
+ std::map<int, int> hmsg_cnt; // count of message per height
+ std::map<int, uint64_t> hmsg_size; // size of message per height
+ std::map<int, uint64_t> hdisk_size; // disk size per height
+ std::map<int, uint64_t> hserial_size; // serial size per height
+} summary_help_extra;
+
+static int summary_helper(BLOCKNUM b, int64_t size, int64_t UU(address), void *extra) {
+ summary_help_extra *CAST_FROM_VOIDP(info, extra);
+ FTNODE n;
+ FTNODE_DISK_DATA ndd = NULL;
+ ftnode_fetch_extra bfe;
+
+ bfe.create_for_full_read(info->ft);
+ int r = toku_deserialize_ftnode_from(info->fd, b, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe);
+ if (r==0) {
+ info->blocksizes += size;
+
+ (info->height_cnt)[n->height]++;
+
+ if (n->height == 0) {
+ info->leafsizes += size;
+ info->leafblocks++;
+ } else {
+ info->nonleafnode_cnt++;
+ }
+
+ info->hdisk_size[n->height] += size;
+ auto serialsize = toku_serialize_ftnode_size(n);
+ info->serialsize += serialsize;
+ info->hserial_size[n->height] += serialsize;
+
+
+ if ((uint64_t)n->height > info->maxheight) {
+ info->maxheight = n->height;
+ }
+
+
+
+ for (int i=0; i<n->n_children; i++) {
+ //printf(" child %d: ", i);
+ if (n->height > 0) {
+ NONLEAF_CHILDINFO bnc = BNC(n, i);
+ unsigned int n_bytes = toku_bnc_nbytesinbuf(bnc);
+ int n_entries = toku_bnc_n_entries(bnc);
+ //if (n_bytes > 0 || n_entries > 0) {
+ // printf(" buffer contains %u bytes (%d items)\n", n_bytes, n_entries);
+ //}
+ info->msg_cnt += n_entries;
+ info->msg_size += n_bytes;
+ info->hmsg_cnt[n->height] += n_entries;
+ info->hmsg_size[n->height] += n_bytes;
+ } else {
+ info->pairs_cnt += BLB_DATA(n, i)->num_klpairs();
+ }
+ }
+ if (n->height ==0) {
+ info->hmsg_cnt[0] += n->n_children; // this way we count partitions per leaf node
+ }
+
+
+ toku_ftnode_free(&n);
+ toku_free(ndd);
+ }
+ return 0;
+}
+
+static std::string humanNumber(uint64_t value) {
+ std::string numWithCommas = to_string(value);
+ int insertPosition = numWithCommas.length() - 3;
+ while (insertPosition > 0) {
+ numWithCommas.insert(insertPosition, ",");
+ insertPosition-=3;
+ }
+ return numWithCommas;
+}
+
+static void dump_summary(int fd, FT ft) {
+ summary_help_extra info;
+ //memset(&info, 0, sizeof(info));
+ info.fd = fd;
+ info.ft = ft;
+ info.blocksizes = 0;
+ info.leafsizes = 0;
+ info.serialsize = 0;
+ info.leafblocks = 0;
+ info.nonleafnode_cnt = 0;
+ info.maxheight = 0;
+ info.msg_cnt = 0;
+ info.msg_size = 0;
+ info.pairs_cnt = 0;
+
+ ft->blocktable.iterate(block_table::TRANSLATION_CHECKPOINTED,
+ summary_helper, &info, true, true);
+ printf("leaf nodes:\t%" PRIu64 "\n", info.leafblocks);
+ printf("non-leaf nodes:\t%" PRIu64 "\n", info.nonleafnode_cnt);
+ printf("Leaf size:\t%s\n", humanNumber(info.leafsizes).c_str());
+ printf("Total size:\t%s\n", humanNumber(info.blocksizes).c_str());
+ printf("Total uncompressed size:\t%s\n", humanNumber(info.serialsize).c_str());
+ printf("Messages count:\t%" PRIu64 "\n", info.msg_cnt);
+ printf("Messages size:\t%s\n", humanNumber(info.msg_size).c_str());
+ printf("Records count:\t%" PRIu64 "\n", info.pairs_cnt);
+ printf("Tree height:\t%" PRIu64 "\n", info.maxheight);
+ for(auto elem : info.height_cnt) {
+ std::string hdr;
+ double children_per_node;
+ if (elem.first == 0) {
+ hdr = "basement nodes";
+ children_per_node = (double)info.hmsg_cnt[0]/elem.second;
+ } else {
+ hdr = "msg cnt";
+ children_per_node = (double)info.height_cnt[elem.first-1]/elem.second;
+ }
+
+ printf("height: %d, nodes count: %d; avg children/node: %f\n\t %s: %d; msg size: %s; disksize: %s; uncompressed size: %s; ratio: %f\n",
+ elem.first, elem.second, children_per_node,
+ hdr.c_str(),
+ info.hmsg_cnt[elem.first],
+ humanNumber(info.hmsg_size[elem.first]).c_str(),
+ humanNumber(info.hdisk_size[elem.first]).c_str(),
+ humanNumber(info.hserial_size[elem.first]).c_str(),
+ (double)info.hserial_size[elem.first]/info.hdisk_size[elem.first] );
+ }
+}
+
+/* ===== end of summary ===== */
+
+static void dump_garbage_stats(int fd, FT ft) {
+ assert(fd == toku_cachefile_get_fd(ft->cf));
+ uint64_t total_space = 0;
+ uint64_t used_space = 0;
+ toku_ft_get_garbage(ft, &total_space, &used_space);
+ printf("garbage total size :%20" PRIu64 "\n", total_space);
+ printf("garbage used size :%20" PRIu64 "\n", used_space);
+ float a=used_space,b=total_space;
+
+ float percentage=((1-a/b)*100);
+ printf("Total garbage : %2.3f%%\n", percentage);
+}
+
+typedef struct __dump_node_extra {
+ int fd;
+ FT ft;
+} dump_node_extra;
+
+static int dump_node_wrapper(BLOCKNUM b, int64_t UU(size), int64_t UU(address), void *extra) {
+ dump_node_extra *CAST_FROM_VOIDP(info, extra);
+ dump_node(info->fd, b, info->ft);
+ return 0;
+}
+
+static uint32_t get_unaligned_uint32(unsigned char *p) {
+ uint32_t n;
+ memcpy(&n, p, sizeof n);
+ return n;
+}
+
+struct dump_sub_block {
+ uint32_t compressed_size;
+ uint32_t uncompressed_size;
+ uint32_t xsum;
+};
+
+static void sub_block_deserialize(struct dump_sub_block *sb, unsigned char *sub_block_header) {
+ sb->compressed_size = toku_dtoh32(get_unaligned_uint32(sub_block_header+0));
+ sb->uncompressed_size = toku_dtoh32(get_unaligned_uint32(sub_block_header+4));
+ sb->xsum = toku_dtoh32(get_unaligned_uint32(sub_block_header+8));
+}
+
+static void verify_block(unsigned char *cp, uint64_t file_offset, uint64_t size) {
+ // verify the header checksum
+ const size_t node_header = 8 + sizeof (uint32_t) + sizeof (uint32_t) + sizeof (uint32_t);
+
+ printf("%.8s layout_version=%u %u build=%d\n", cp, get_unaligned_uint32(cp+8), get_unaligned_uint32(cp+12), get_unaligned_uint32(cp+16));
+
+ unsigned char *sub_block_header = &cp[node_header];
+ uint32_t n_sub_blocks = toku_dtoh32(get_unaligned_uint32(&sub_block_header[0]));
+ uint32_t header_length = node_header + n_sub_blocks * sizeof (struct dump_sub_block);
+ header_length += sizeof (uint32_t); // CRC
+ if (header_length > size) {
+ printf("header length too big: %u\n", header_length);
+ return;
+ }
+ uint32_t header_xsum = toku_x1764_memory(cp, header_length);
+ uint32_t expected_xsum = toku_dtoh32(get_unaligned_uint32(&cp[header_length]));
+ if (header_xsum != expected_xsum) {
+ printf("header checksum failed: %u %u\n", header_xsum, expected_xsum);
+ return;
+ }
+
+ // deserialize the sub block header
+ struct dump_sub_block sub_block[n_sub_blocks];
+ sub_block_header += sizeof (uint32_t);
+ for (uint32_t i = 0 ; i < n_sub_blocks; i++) {
+ sub_block_deserialize(&sub_block[i], sub_block_header);
+ sub_block_header += sizeof (struct dump_sub_block);
+ }
+
+ // verify the sub block header
+ uint32_t offset = header_length + 4;
+ for (uint32_t i = 0 ; i < n_sub_blocks; i++) {
+ uint32_t xsum = toku_x1764_memory(cp + offset, sub_block[i].compressed_size);
+ printf("%u: %u %u %u", i, sub_block[i].compressed_size, sub_block[i].uncompressed_size, sub_block[i].xsum);
+ if (xsum != sub_block[i].xsum)
+ printf(" fail %u offset %" PRIu64, xsum, file_offset + offset);
+ printf("\n");
+ offset += sub_block[i].compressed_size;
+ }
+ if (offset != size)
+ printf("offset %u expected %" PRIu64 "\n", offset, size);
+}
+
+static void dump_block(int fd, BLOCKNUM blocknum, FT ft) {
+ DISKOFF offset, size;
+ ft->blocktable.translate_blocknum_to_offset_size(blocknum, &offset, &size);
+ printf("%" PRId64 " at %" PRId64 " size %" PRId64 "\n", blocknum.b, offset, size);
+
+ unsigned char *CAST_FROM_VOIDP(vp, toku_malloc(size));
+ uint64_t r = pread(fd, vp, size, offset);
+ if (r == (uint64_t)size) {
+ verify_block(vp, offset, size);
+ }
+ toku_free(vp);
+}
+
+static void dump_file(int fd, uint64_t offset, uint64_t size, FILE *outfp) {
+ unsigned char *XMALLOC_N(size, vp);
+ uint64_t r = pread(fd, vp, size, offset);
+ if (r == size) {
+ if (outfp == stdout) {
+ hex_dump(vp, offset, size);
+ } else {
+ size_t wrote = fwrite(vp, size, 1, outfp);
+ assert(wrote == 1);
+ }
+ }
+ toku_free(vp);
+}
+
+static void set_file(int fd, uint64_t offset, unsigned char newc) {
+ toku_os_pwrite(fd, &newc, sizeof newc, offset);
+}
+
+static int readline(char *line, int maxline) {
+ int i = 0;
+ int c;
+ while ((c = getchar()) != EOF && c != '\n' && i < maxline) {
+ line[i++] = (char)c;
+ }
+ line[i++] = 0;
+ return c == EOF ? EOF : i;
+}
+
+static int split_fields(char *line, char *fields[], int maxfields) {
+ int i;
+ for (i=0; i<maxfields; i++)
+ fields[i] = NULL;
+ for (i=0; i<maxfields; i++, line=NULL) {
+ fields[i] = strtok(line, " ");
+ if (fields[i] == NULL) {
+ break;
+ }
+ }
+ return i;
+}
+
+static uint64_t getuint64(const char *f) {
+ if (strncmp(f, "0x", 2) == 0 || strncmp(f, "0X", 2) == 0)
+ return strtoull(f, 0, 16);
+ else if (strncmp(f, "0", 1) == 0)
+ return strtoull(f, 0, 8);
+ else
+ return strtoull(f, 0, 10);
+}
+
+static void interactive_help(void) {
+ fprintf(stderr, "help\n");
+ fprintf(stderr, "header\n");
+ cout <<"mr/MessagesReport [NUMBER] \n Reports messages for the level of the tree you want get more details about\n";
+ cout <<"rf/readFile ft-file-name \n Switch to a different FT\n";
+ fprintf(stderr, "node NUMBER \n");
+ fprintf(stderr, "bx OFFSET | block_translation OFFSET\n");
+ fprintf(stderr, "dumpdata 0|1\n");
+ fprintf(stderr, "fragmentation\n");
+ fprintf(stderr, "nodesizes\n");
+ fprintf(stderr, "garbage\n");
+ fprintf(stderr, "file OFFSET SIZE [outfilename]\n");
+ fprintf(stderr, "quit\n");
+}
+
+static void freeNMC(NMC *msgs[], int height){
+ for(int i=0;i<height;i++){
+ if(msgs[i]!=NULL){
+ delete(msgs[i]->count);
+
+ while(msgs[i]->nextNode!=NULL){
+ NMC* ptr=msgs[i]->nextNode;
+ msgs[i]=msgs[i]->nextNode;
+ delete ptr;
+
+ }
+ msgs[i]=NULL;
+ }
+ }
+}
+
+static void writeTree(NMC *msgs[],int height,char *name UU()){
+ ofstream mytree ("/tmp/tree.txt",fstream::out);
+ if (mytree.is_open()){
+ for(int i=height;i>=0;i--){
+ NMC * ptr=msgs[i];
+ mytree <<i<<endl;
+ while(ptr!=NULL){
+ mytree << ptr->id<<"\t";
+ if(ptr->clean!=0)mytree << "1"<<"\t";
+ else mytree << "0"<<"\t";
+ for(int j=0;j<15;j++)mytree << ptr->count[j]<<" ";
+ mytree << ptr->count[i]<<endl;
+ ptr=ptr->nextNode;
+ }
+ mytree <<endl;
+ }
+ }
+ else cout << "Unable to open file";
+ mytree.close();
+}
+
+static void writeJson(NMC *msgs[],int height,const char *name){
+ ofstream mytree (name,fstream::out);
+ if (mytree.is_open()){
+ mytree <<"{\n \"FT\":[";
+ for(int i=height;i>=0;i--){
+ NMC * ptr=msgs[i];
+ mytree <<"{\n\"Level\": {\"Height\":\""<<i<<"\",\n \"Nodes\":[";
+ while(ptr!=NULL){
+ mytree <<"{\"ID\":\""<< ptr->id<<"\",";
+ if(ptr->clean!=0){
+ mytree <<"\"Messages\":[";
+ for(int j=0;j<16;j++)
+ {
+ mytree <<"{";
+ switch (j) {
+ case FT_INSERT: mytree <<"\"INSERT\":\""<<ptr->count[j]<<"\""; break;
+ case FT_INSERT_NO_OVERWRITE: mytree <<"\"INSERT_NOVERWTE\":\""<<ptr->count[j]<<"\""; break;
+ case FT_DELETE_ANY: mytree <<"\"DELETE\":\""<<ptr->count[j]<<"\""; break;
+ case FT_ABORT_ANY: mytree <<"\"ABORT\":\""<<ptr->count[j]<<"\""; break;
+ case FT_COMMIT_ANY: mytree <<"\"COMMITY\":\""<<ptr->count[j]<<"\""; break;
+ case FT_COMMIT_BROADCAST_ALL: mytree <<"\"COMMIT_BROADCAST_ALL\":\""<<ptr->count[j]<<"\"" ; break;
+ case FT_COMMIT_BROADCAST_TXN: mytree <<"\"COMMIT_BROADCAST_TXN\":\""<<ptr->count[j]<<"\""; break;
+ case FT_ABORT_BROADCAST_TXN: mytree <<"\"ABORT_BROADCAST_TXN\":\""<<ptr->count[j]<<"\"";break;
+ case FT_OPTIMIZE: mytree <<"\"OPTIMIZE\":\""<<ptr->count[j]<<"\""; break;
+ case FT_OPTIMIZE_FOR_UPGRADE: mytree <<"\"OPTIMIZE_FOR_UPGRADE\":\""<<ptr->count[j]<<"\"";break;
+ case FT_UPDATE: mytree <<"\"UPDATE\":\""<<ptr->count[j]<<"\""; break;
+ case FT_UPDATE_BROADCAST_ALL: mytree <<"\"UPDATE_BROADCAST_ALL\":\""<<ptr->count[j]<<"\""; break;
+ }
+ mytree <<"}";
+ if(j<15)mytree<<",";
+ }
+
+ mytree <<"]}";
+
+ }
+ else {
+ mytree <<"\"Messages\":\""<< "0"<<"\"}";
+ }
+ if(ptr->nextNode!=NULL)mytree <<",\n";
+ else mytree <<"]}\n";
+ ptr=ptr->nextNode;
+ }
+ mytree <<"\n}\n";
+ if(i!=0)mytree <<",\n";
+ }
+ mytree <<"\n]}\n";
+
+ }
+ else cout << "Unable to open file";
+ mytree.close();
+}
+
+static void writeTree(NMC *msgs[],int height){
+ ofstream mytree ("/tmp/tree1.txt",fstream::out);
+ if (mytree.is_open()){
+ for(int i=height;i>=0;i--){
+ NMC * ptr=msgs[i];
+ mytree <<i<<endl;
+ while(ptr!=NULL){
+ mytree << ptr->id<<",";
+ if(ptr->clean!=0)mytree << "1"<<",";
+ else mytree << "0"<<",";
+ for(int j=0;j<15;j++)mytree << ptr->count[j]<<",";
+ mytree << ptr->count[i]<<endl;
+ ptr=ptr->nextNode;
+ }
+ mytree <<".\"";
+ }
+ }
+ else cout << "Unable to open file";
+ mytree.close();
+}
+
+static void FT_to_JSON(int fd, FT ft, CACHEFILE cf, const char * JsonFile){
+ toku_ft_free(ft);
+ open_header(fd, &ft, cf);
+ int root=getRootNode(ft);
+ BLOCKNUM off = make_blocknum(root);
+ int height=getHeight(fd,off, ft);
+ NMC *msgs[height];
+ for(int i=0;i<=height;i++){
+ msgs[i]=NULL;
+ }
+ open_header(fd, &ft, cf);
+ root=getRootNode(ft);
+ off = make_blocknum(root);
+ countMessagesInFT(fd,off, ft,msgs);
+ cout <<"to STD output: \n";
+ treeToSTDout(msgs,height);
+ writeTree(msgs,height);
+ cout<<"FT's json file was generated here:";
+ if(JsonFile!=NULL) {
+ cout <<JsonFile;
+ writeJson(msgs,height,JsonFile);
+ }
+ else {
+ cout <<"./FT.json";
+ writeJson(msgs,height,"./FT.json");
+ }
+ cout<<endl;
+ freeNMC(msgs,height);
+ exit(0);
+}
+
+static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) {
+ toku_ft_free(ft);
+ open_header(fd, &ft, cf);
+ int root=getRootNode(ft);
+ BLOCKNUM off = make_blocknum(root);
+ int height=getHeight(fd,off, ft);
+ NMC *msgs[height];
+ for(int i=0;i<=height;i++){
+ msgs[i]=NULL;
+ }
+ while (1) {
+ printf("ftdump>");
+ fflush(stdout);
+ char line[maxline+1];
+ int r = readline(line, maxline);
+ if (r == EOF)
+ break;
+ const int maxfields = 4;
+ char *fields[maxfields];
+ int nfields = split_fields(line, fields, maxfields);
+ if (nfields == 0)
+ continue;
+ if (strcmp(fields[0], "help") == 0) {
+ interactive_help();
+ } else if (strcmp(fields[0], "header") == 0) {
+ toku_ft_free(ft);
+ open_header(fd, &ft, cf);
+ dump_header(ft);
+ } else if (strcmp(fields[0], "rn") == 0||strcmp(fields[0], "rootNode")==0||strcmp(fields[0], "rootnode") == 0) {
+ printf("Root node :%d\n",root);
+ } else if (strcmp(fields[0], "block") == 0 && nfields == 2) {
+ BLOCKNUM blocknum = make_blocknum(getuint64(fields[1]));
+ dump_block(fd, blocknum, ft);
+ }else if ((strcmp(fields[0], "readFile") == 0 ||strcmp(fields[0], "readfile") == 0 ||strcmp(fields[0], "rf") == 0 )&& nfields == 2) {
+ fname=fields[1];
+ fd = open(fname, O_RDWR + O_BINARY);
+ toku_ft_free(ft);
+ open_header(fd, &ft, cf);
+ root=getRootNode(ft);
+ off = make_blocknum(root);
+ height=getHeight(fd,off, ft);
+ if (fd < 0) {
+ fprintf(stderr, "%s: can not open the FT dump %s errno %d\n", arg0, fname, errno);
+ continue;
+ }
+ } else if (strcmp(fields[0], "node") == 0 && nfields == 2) {
+ off = make_blocknum(getuint64(fields[1]));
+ dump_node(fd, off, ft);
+ }else if ((strcmp(fields[0], "mr") == 0||(strcmp(fields[0], "nc")) == 0 ||strcmp(fields[0], "messagesReport") == 0 )) {
+ freeNMC(msgs,height);
+ toku_ft_free(ft);
+ open_header(fd, &ft, cf);
+ root=getRootNode(ft);
+ off = make_blocknum(root);
+ countMessagesInFT(fd,off, ft,msgs);
+ int level=-1;
+ if(nfields == 2)level=getuint64(fields[1]);
+ if(level>=0){
+ levelToSTDout(msgs[level], level);
+ }
+ else{
+ cout <<"to STD output: \n";
+ treeToSTDout(msgs,height);
+ }
+ writeTree(msgs,height);
+ writeTree(msgs,height, NULL);
+
+ }else if (strcmp(fields[0], "dumpdata") == 0 && nfields == 2) {
+
+ do_dump_data = strtol(fields[1], NULL, 10);
+ }
+ else if (strcmp(fields[0], "block_translation") == 0 || strcmp(fields[0], "bx") == 0) {
+ uint64_t offset = 0;
+ if (nfields == 2)
+ offset = getuint64(fields[1]);
+ dump_block_translation(ft, offset);
+ } else if (strcmp(fields[0], "fragmentation") == 0) {
+ dump_fragmentation(fd, ft, do_tsv);
+ } else if (strcmp(fields[0], "nodesizes") == 0) {
+ dump_nodesizes(fd, ft);
+ } else if (strcmp(fields[0], "garbage") == 0||strcmp(fields[0], "g") == 0) {
+ dump_garbage_stats(fd, ft);
+ } else if (strcmp(fields[0], "file") == 0 && nfields >= 3) {
+ uint64_t offset = getuint64(fields[1]);
+ uint64_t size = getuint64(fields[2]);
+ FILE *outfp = stdout;
+ if (nfields >= 4)
+ outfp = fopen(fields[3], "w");
+ dump_file(fd, offset, size, outfp);
+ } else if (strcmp(fields[0], "setfile") == 0 && nfields == 3) {
+ uint64_t offset = getuint64(fields[1]);
+ unsigned char newc = getuint64(fields[2]);
+ set_file(fd, offset, newc);
+ } else if (strcmp(fields[0], "quit") == 0 || strcmp(fields[0], "q") == 0) {
+ toku_ft_free(ft);
+ exit(0);
+ }
+ }
+ freeNMC(msgs,height);
+}
+
+static int usage(void) {
+ fprintf(stderr, "Usage: %s ", arg0);
+ fprintf(stderr, "--interactive ");
+ fprintf(stderr, "--support /path/to/fractal-tree/file \n\t an interactive way to see what messages and/or switch between FTs");
+ fprintf(stderr, "--json /path/to/fractal-tree/file [output json file]\n\t if left empty an FT.json will be created automatically");
+ fprintf(stderr, "--nodata ");
+ fprintf(stderr, "--dumpdata 0|1 ");
+ fprintf(stderr, "--header ");
+ fprintf(stderr, "--rootnode ");
+ fprintf(stderr, "--node N ");
+ fprintf(stderr, "--fragmentation ");
+ fprintf(stderr, "--garbage ");
+ fprintf(stderr, "--tsv ");
+ fprintf(stderr, "--translation-table ");
+ fprintf(stderr, "--tsv ");
+ fprintf(stderr, "--summary ");
+ fprintf(stderr, "filename \n");
+ return 1;
+}
+
+int main (int argc, const char *const argv[]) {
+ arg0 = argv[0];
+ argc--; argv++;
+ while (argc>0) {
+ if (strcmp(argv[0], "--interactive") == 0 || strcmp(argv[0], "--i") == 0) {
+ do_interactive = 1;
+ }
+ else if ((strcmp(argv[0], "--json") == 0 || strcmp(argv[0], "--s")== 0)&& argc >= 2) {
+ do_json = 1;
+ fname=argv[1];
+ argc--; argv++;
+ break;
+ } else if (strcmp(argv[0], "--nodata") == 0) {
+ do_dump_data = 0;
+ } else if (strcmp(argv[0], "--dumpdata") == 0 && argc > 1) {
+ do_dump_data = atoi(argv[0]);
+ } else if (strcmp(argv[0], "--header") == 0) {
+ do_header = 1;
+ } else if (strcmp(argv[0], "--rootnode") == 0) {
+ do_rootnode = 1;
+ } else if (strcmp(argv[0], "--node") == 0 && argc > 1) {
+ argc--; argv++;
+ do_node = 1;
+ do_node_num = make_blocknum(getuint64(argv[0]));
+ } else if (strcmp(argv[0], "--fragmentation") == 0) {
+ do_fragmentation = 1;
+ } else if (strcmp(argv[0], "--garbage") == 0) {
+ do_garbage = 1;
+ } else if (strcmp(argv[0], "--tsv") == 0) {
+ do_tsv = 1;
+ } else if (strcmp(argv[0], "--translation-table") == 0) {
+ do_translation_table = 1;
+ } else if (strcmp(argv[0], "--summary") == 0) {
+ do_summary = 1;
+ } else if (strcmp(argv[0], "--help") == 0 || strcmp(argv[0], "-?") == 0 || strcmp(argv[0], "-h") == 0) {
+ return usage();
+ } else {
+ break;
+ }
+ argc--; argv++;
+ }
+ if (argc != 1 && do_json==0)
+ return usage();
+
+ int r = toku_ft_layer_init();
+ assert_zero(r);
+ if(fname==NULL)fname = argv[0];
+ int fd = open(fname, O_RDWR + O_BINARY);
+ if (fd < 0) {
+ fprintf(stderr, "%s: can not open %s errno %d\n", arg0, fname, errno);
+ return 1;
+ }
+ // create a cachefile for the header
+ CACHETABLE ct = NULL;
+ toku_cachetable_create(&ct, 1<<25, (LSN){0}, 0);
+ CACHEFILE cf = NULL;
+ r = toku_cachetable_openfd (&cf, ct, fd, fname);
+ assert_zero(r);
+ FT ft = NULL;
+ open_header(fd, &ft, cf);
+ if (do_json ) {
+ const char *arg=argv[1];
+ FT_to_JSON(fd, ft, cf,arg);
+ }
+ if (do_interactive) {
+ run_iteractive_loop(fd, ft, cf);
+ }
+ else {
+ if (do_header) {
+ dump_header(ft);
+ }
+ if (do_rootnode) {
+ dump_node(fd, ft->h->root_blocknum, ft);
+ }
+ if (do_node) {
+ dump_node(fd, do_node_num, ft);
+ }
+ if (do_fragmentation) {
+ dump_fragmentation(fd, ft, do_tsv);
+ }
+ if (do_translation_table) {
+ ft->blocktable.dump_translation_table_pretty(stdout);
+ }
+ if (do_summary) {
+ dump_summary(fd, ft);
+ }
+ if (do_garbage) {
+ dump_garbage_stats(fd, ft);
+ }
+ if (!do_header && !do_rootnode && !do_fragmentation && !do_translation_table && !do_garbage && !do_summary) {
+ printf("Block translation:");
+ ft->blocktable.dump_translation_table(stdout);
+ dump_header(ft);
+ struct __dump_node_extra info;
+ info.fd = fd;
+ info.ft = ft;
+ ft->blocktable.iterate(block_table::TRANSLATION_CHECKPOINTED,
+ dump_node_wrapper, &info, true, true);
+ }
+ }
+ toku_cachefile_close(&cf, false, ZERO_LSN);
+ toku_cachetable_close(&ct);
+ toku_ft_free(ft);
+ toku_ft_layer_destroy();
+ return 0;
+}