diff options
Diffstat (limited to 'storage/tokudb/PerconaFT/tools')
| -rw-r--r-- | storage/tokudb/PerconaFT/tools/CMakeLists.txt | 25 | ||||
| -rw-r--r-- | storage/tokudb/PerconaFT/tools/ftverify.cc | 452 | ||||
| -rw-r--r-- | storage/tokudb/PerconaFT/tools/pmprof | 31 | ||||
| -rw-r--r-- | storage/tokudb/PerconaFT/tools/tdb-recover.cc | 80 | ||||
| -rw-r--r-- | storage/tokudb/PerconaFT/tools/tokudb_dump.cc | 685 | ||||
| -rw-r--r-- | storage/tokudb/PerconaFT/tools/tokuft_logprint.cc | 74 | ||||
| -rw-r--r-- | storage/tokudb/PerconaFT/tools/tokuftdump.cc | 1246 |
7 files changed, 2593 insertions, 0 deletions
diff --git a/storage/tokudb/PerconaFT/tools/CMakeLists.txt b/storage/tokudb/PerconaFT/tools/CMakeLists.txt new file mode 100644 index 00000000..dd54249a --- /dev/null +++ b/storage/tokudb/PerconaFT/tools/CMakeLists.txt @@ -0,0 +1,25 @@ +set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE DONT_DEPRECATE_ERRNO) + +set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify) +foreach(tool ${tools}) + add_executable(${tool} ${tool}.cc) + add_dependencies(${tool} install_tdb_h) + target_link_libraries(${tool} ${LIBTOKUDB}_static ft_static z lzma snappy ${LIBTOKUPORTABILITY}_static ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_SYSTEM_LIBS}) + + # detect when we are being built as a subproject + if (DEFINED MYSQL_PROJECT_NAME_DOCSTRING) + if ((CMAKE_BUILD_TYPE MATCHES "Debug") AND + (CMAKE_CXX_FLAGS_DEBUG MATCHES " -DENABLED_DEBUG_SYNC")) + target_link_libraries(${tool} sql) + endif() + target_link_libraries(${tool} mysys) + endif () + + add_space_separated_property(TARGET ${tool} COMPILE_FLAGS -fvisibility=hidden) +endforeach(tool) + +# link in math.h library just for this tool. +target_link_libraries(ftverify m) + +install(TARGETS tokuftdump DESTINATION ${INSTALL_BINDIR} COMPONENT tokudb-engine) +install(TARGETS tokuft_logprint DESTINATION ${INSTALL_BINDIR} COMPONENT tokudb-engine) diff --git a/storage/tokudb/PerconaFT/tools/ftverify.cc b/storage/tokudb/PerconaFT/tools/ftverify.cc new file mode 100644 index 00000000..ee40b991 --- /dev/null +++ b/storage/tokudb/PerconaFT/tools/ftverify.cc @@ -0,0 +1,452 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +//////////////////////////////////////////////////////////////////// +// ftverify - Command line tool that checks the validity of a given +// fractal tree file, one block at a time. +//////////////////////////////////////////////////////////////////// + +#include "portability/toku_assert.h" +#include "portability/toku_list.h" +#include "portability/toku_portability.h" + +#include "ft/serialize/block_allocator.h" +#include "ft/ft-internal.h" +#include "ft/serialize/ft-serialize.h" +#include "ft/serialize/ft_layout_version.h" +#include "ft/serialize/ft_node-serialize.h" +#include "ft/node.h" +#include "ft/serialize/rbuf.h" +#include "ft/serialize/sub_block.h" +#include "util/threadpool.h" + +#include <fcntl.h> +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sysexits.h> +#include <unistd.h> + +static int num_cores = 0; // cache the number of cores for the parallelization +static struct toku_thread_pool *ft_pool = NULL; +static FILE *outf; +static double pct = 0.5; + +// Struct for reporting sub block stats. +struct verify_block_extra { + BLOCKNUM b; + int n_sub_blocks; + uint32_t header_length; + uint32_t calc_xsum; + uint32_t stored_xsum; + bool header_valid; + bool sub_blocks_valid; + struct sub_block_info *sub_block_results; +}; + +// Initialization function for the sub block stats. +static void +init_verify_block_extra(BLOCKNUM b, struct verify_block_extra *e) +{ + static const struct verify_block_extra default_vbe = + { + .b = { 0 }, + .n_sub_blocks = 0, + .header_length = 0, + .calc_xsum = 0, + .stored_xsum = 0, + .header_valid = true, + .sub_blocks_valid = true, + .sub_block_results = NULL + }; + *e = default_vbe; + e->b = b; +} + +// Reports percentage of completed blocks. +static void +report(int64_t blocks_done, int64_t blocks_failed, int64_t total_blocks) +{ + int64_t blocks_per_report = llrint(pct * total_blocks / 100.0); + if (blocks_per_report < 1) { + blocks_per_report = 1; + } + if (blocks_done % blocks_per_report == 0) { + double pct_actually_done = (100.0 * blocks_done) / total_blocks; + printf("% 3.3lf%% | %" PRId64 " blocks checked, %" PRId64 " bad block(s) detected\n", + pct_actually_done, blocks_done, blocks_failed); + fflush(stdout); + } +} + +// Helper function to deserialize one of the two headers for the ft +// we are checking. +static void +deserialize_headers(int fd, struct ft **h1p, struct ft **h2p) +{ + struct rbuf rb_0; + struct rbuf rb_1; + uint64_t checkpoint_count_0; + uint64_t checkpoint_count_1; + LSN checkpoint_lsn_0; + LSN checkpoint_lsn_1; + uint32_t version_0, version_1; + bool h0_acceptable = false; + bool h1_acceptable = false; + int r0, r1; + int r; + + { + toku_off_t header_0_off = 0; + r0 = deserialize_ft_from_fd_into_rbuf( + fd, + header_0_off, + &rb_0, + &checkpoint_count_0, + &checkpoint_lsn_0, + &version_0 + ); + if ((r0==0) && (checkpoint_lsn_0.lsn <= MAX_LSN.lsn)) { + h0_acceptable = true; + } + } + { + toku_off_t header_1_off = BlockAllocator::BLOCK_ALLOCATOR_HEADER_RESERVE; + r1 = deserialize_ft_from_fd_into_rbuf( + fd, + header_1_off, + &rb_1, + &checkpoint_count_1, + &checkpoint_lsn_1, + &version_1 + ); + if ((r1==0) && (checkpoint_lsn_1.lsn <= MAX_LSN.lsn)) { + h1_acceptable = true; + } + } + + // If either header is too new, the dictionary is unreadable + if (r0 == TOKUDB_DICTIONARY_TOO_NEW || r1 == TOKUDB_DICTIONARY_TOO_NEW) { + fprintf(stderr, "This dictionary was created with a version of PerconaFT that is too new. Aborting.\n"); + abort(); + } + if (h0_acceptable) { + printf("Found dictionary header 1 with LSN %" PRIu64 "\n", checkpoint_lsn_0.lsn); + r = deserialize_ft_versioned(fd, &rb_0, h1p, version_0); + + if (r != 0) { + printf("---Header Error----\n"); + } + + } else { + *h1p = NULL; + } + if (h1_acceptable) { + printf("Found dictionary header 2 with LSN %" PRIu64 "\n", checkpoint_lsn_1.lsn); + r = deserialize_ft_versioned(fd, &rb_1, h2p, version_1); + if (r != 0) { + printf("---Header Error----\n"); + } + } else { + *h2p = NULL; + } + + if (rb_0.buf) toku_free(rb_0.buf); + if (rb_1.buf) toku_free(rb_1.buf); +} + +// Helper struct for tracking block checking progress. +struct check_block_table_extra { + int fd; + int64_t blocks_done, blocks_failed, total_blocks; + struct ft *h; +}; + +// Check non-upgraded (legacy) node. +// NOTE: These nodes have less checksumming than more +// recent nodes. This effectively means that we are +// skipping over these nodes. +static int +check_old_node(FTNODE node, struct rbuf *rb, int version) +{ + int r = 0; + read_legacy_node_info(node, rb, version); + // For version 14 nodes, advance the buffer to the end + // and verify the checksum. + if (version == FT_FIRST_LAYOUT_VERSION_WITH_END_TO_END_CHECKSUM) { + // Advance the buffer to the end. + rb->ndone = rb->size - 4; + r = check_legacy_end_checksum(rb); + } + + return r; +} + +// Read, decompress, and check the given block. +static int +check_block(BLOCKNUM blocknum, int64_t UU(blocksize), int64_t UU(address), void *extra) +{ + int r = 0; + int failure = 0; + struct check_block_table_extra *CAST_FROM_VOIDP(cbte, extra); + int fd = cbte->fd; + FT ft = cbte->h; + + struct verify_block_extra be; + init_verify_block_extra(blocknum, &be); + + // Let's read the block off of disk and fill a buffer with that + // block. + struct rbuf rb = RBUF_INITIALIZER; + read_block_from_fd_into_rbuf(fd, blocknum, ft, &rb); + + // Allocate the node. + FTNODE XMALLOC(node); + + initialize_ftnode(node, blocknum); + + r = read_and_check_magic(&rb); + if (r == DB_BADFORMAT) { + printf(" Magic failed.\n"); + failure++; + } + + r = read_and_check_version(node, &rb); + if (r != 0) { + printf(" Version check failed.\n"); + failure++; + } + + int version = node->layout_version_read_from_disk; + + //////////////////////////// + // UPGRADE FORK GOES HERE // + //////////////////////////// + + // Check nodes before major layout changes in version 15. + // All newer versions should follow the same layout, for now. + // This predicate would need to be changed if the layout + // of the nodes on disk does indeed change in the future. + if (version < FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES) + { + struct rbuf nrb; + // Use old decompression method for legacy nodes. + r = decompress_from_raw_block_into_rbuf(rb.buf, rb.size, &nrb, blocknum); + if (r != 0) { + failure++; + goto cleanup; + } + + // Check the end-to-end checksum. + r = check_old_node(node, &nrb, version); + if (r != 0) { + failure++; + } + goto cleanup; + } + + read_node_info(node, &rb, version); + + FTNODE_DISK_DATA ndd; + allocate_and_read_partition_offsets(node, &rb, &ndd); + + r = check_node_info_checksum(&rb); + if (r == TOKUDB_BAD_CHECKSUM) { + printf(" Node info checksum failed.\n"); + failure++; + } + + // Get the partition info sub block. + struct sub_block sb; + sub_block_init(&sb); + r = read_compressed_sub_block(&rb, &sb); + if (r != 0) { + printf(" Partition info checksum failed.\n"); + failure++; + } + + just_decompress_sub_block(&sb); + + // If we want to inspect the data inside the partitions, we need + // to call setup_ftnode_partitions(node, bfe, true) + + // <CER> TODO: Create function for this. + // Using the node info, decompress all the keys and pivots to + // detect any corruptions. + for (int i = 0; i < node->n_children; ++i) { + uint32_t curr_offset = BP_START(ndd,i); + uint32_t curr_size = BP_SIZE(ndd,i); + struct rbuf curr_rbuf = {.buf = NULL, .size = 0, .ndone = 0}; + rbuf_init(&curr_rbuf, rb.buf + curr_offset, curr_size); + struct sub_block curr_sb; + sub_block_init(&curr_sb); + + r = read_compressed_sub_block(&rb, &sb); + if (r != 0) { + printf(" Compressed child partition %d checksum failed.\n", i); + failure++; + } + just_decompress_sub_block(&sb); + + r = verify_ftnode_sub_block(&sb, nullptr, blocknum); + if (r != 0) { + printf(" Uncompressed child partition %d checksum failed.\n", i); + failure++; + } + + // <CER> If needed, we can print row and/or pivot info at this + // point. + } + +cleanup: + // Cleanup and error incrementing. + if (failure) { + cbte->blocks_failed++; + } + + cbte->blocks_done++; + + if (node) { + toku_free(node); + } + + // Print the status of this block to the console. + report(cbte->blocks_done, cbte->blocks_failed, cbte->total_blocks); + // We need to ALWAYS return 0 if we want to continue iterating + // through the nodes in the file. + r = 0; + return r; +} + +// This calls toku_blocktable_iterate on the given block table. +// Passes our check_block() function to be called as we iterate over +// the block table. This will print any interesting failures and +// update us on our progress. +static void check_block_table(int fd, block_table *bt, struct ft *h) { + int64_t num_blocks = bt->get_blocks_in_use_unlocked(); + printf("Starting verification of checkpoint containing"); + printf(" %" PRId64 " blocks.\n", num_blocks); + fflush(stdout); + + struct check_block_table_extra extra = { .fd = fd, + .blocks_done = 0, + .blocks_failed = 0, + .total_blocks = num_blocks, + .h = h }; + int r = bt->iterate(block_table::TRANSLATION_CURRENT, + check_block, + &extra, + true, + true); + if (r != 0) { + // We can print more information here if necessary. + } + + assert(extra.blocks_done == extra.total_blocks); + printf("Finished verification. "); + printf(" %" PRId64 " blocks checked,", extra.blocks_done); + printf(" %" PRId64 " bad block(s) detected\n", extra.blocks_failed); + fflush(stdout); +} + +int +main(int argc, char const * const argv[]) +{ + // open the file + int r = 0; + int dictfd; + const char *dictfname, *outfname; + if (argc < 3 || argc > 4) { + fprintf(stderr, "%s: Invalid arguments.\n", argv[0]); + fprintf(stderr, "Usage: %s <dictionary> <logfile> [report%%]\n", argv[0]); + r = EX_USAGE; + goto exit; + } + + assert(argc == 3 || argc == 4); + dictfname = argv[1]; + outfname = argv[2]; + if (argc == 4) { + set_errno(0); + pct = strtod(argv[3], NULL); + assert_zero(get_maybe_error_errno()); + assert(pct > 0.0 && pct <= 100.0); + } + + // Open the file as read-only. + dictfd = open(dictfname, O_RDONLY | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); + if (dictfd < 0) { + perror(dictfname); + fflush(stderr); + abort(); + } + outf = fopen(outfname, "w"); + if (!outf) { + perror(outfname); + fflush(stderr); + abort(); + } + + // body of toku_ft_serialize_init(); + num_cores = toku_os_get_number_active_processors(); + r = toku_thread_pool_create(&ft_pool, num_cores); lazy_assert_zero(r); + assert_zero(r); + + // deserialize the header(s) + struct ft *h1, *h2; + deserialize_headers(dictfd, &h1, &h2); + + // walk over the block table and check blocks + if (h1) { + printf("Checking dictionary from header 1.\n"); + check_block_table(dictfd, &h1->blocktable, h1); + } + if (h2) { + printf("Checking dictionary from header 2.\n"); + check_block_table(dictfd, &h2->blocktable, h2); + } + if (h1 == NULL && h2 == NULL) { + printf("Both headers have a corruption and could not be used.\n"); + } + + toku_thread_pool_destroy(&ft_pool); +exit: + return r; +} diff --git a/storage/tokudb/PerconaFT/tools/pmprof b/storage/tokudb/PerconaFT/tools/pmprof new file mode 100644 index 00000000..de0a7e3c --- /dev/null +++ b/storage/tokudb/PerconaFT/tools/pmprof @@ -0,0 +1,31 @@ +#!/bin/bash + +# a poor man's profiler +# http://webcache.googleusercontent.com/search?q=cache:http://mituzas.lt/2009/02/15/poor-mans-contention-profiling/ + +nsamples=1 +sleeptime=1 + +while [ $# -gt 0 ] ; do + arg=$1; + if [[ $arg =~ --(.*)=(.*) ]] ; then + eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]} + else + break + fi + shift +done + +pid=$1 + +for x in $(seq 1 $nsamples) + do + gdb -ex "set pagination 0" -ex "thread apply all bt" -batch -p $pid + sleep $sleeptime + done | \ +awk ' + BEGIN { s = ""; } + /^Thread/ { if (s != "") print s; s = ""; } + /^\#/ { if ($3 == "in") { v = $4; } else { v = $2 } if (s != "" ) { s = s "," v} else { s = v } } + END { print s }' | \ +sort | uniq -c | sort -r -n -k 1,1 diff --git a/storage/tokudb/PerconaFT/tools/tdb-recover.cc b/storage/tokudb/PerconaFT/tools/tdb-recover.cc new file mode 100644 index 00000000..f01d0109 --- /dev/null +++ b/storage/tokudb/PerconaFT/tools/tdb-recover.cc @@ -0,0 +1,80 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +/* Recover an env. The logs are in argv[1]. The new database is created in the cwd. */ + +// Test: +// cd ../src/tests/tmpdir +// ../../../ft/recover ../dir.test_log2.c.tdb + +#include "ft/ft-ops.h" +#include "ft/logger/recover.h" + +static int recovery_main(int argc, const char *const argv[]); + +int main(int argc, const char *const argv[]) { + int r = toku_ft_layer_init(); + assert(r == 0); + r = recovery_main(argc, argv); + toku_ft_layer_destroy(); + return r; +} + +int recovery_main (int argc, const char *const argv[]) { + const char *data_dir, *log_dir; + if (argc==3) { + data_dir = argv[1]; + log_dir = argv[2]; + } else if (argc==2) { + data_dir = log_dir = argv[1]; + } else { + printf("Usage: %s <datadir> [ <logdir> ]\n", argv[0]); + return(1); + } + + int r = tokuft_recover(nullptr, + nullptr, + nullptr, + nullptr, + data_dir, log_dir, nullptr, nullptr, nullptr, nullptr, 0); + if (r!=0) { + fprintf(stderr, "Recovery failed\n"); + return(1); + } + return 0; +} diff --git a/storage/tokudb/PerconaFT/tools/tokudb_dump.cc b/storage/tokudb/PerconaFT/tools/tokudb_dump.cc new file mode 100644 index 00000000..d7362fc6 --- /dev/null +++ b/storage/tokudb/PerconaFT/tools/tokudb_dump.cc @@ -0,0 +1,685 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <db.h> + +#include <toku_stdlib.h> +#include <toku_stdint.h> +#include <toku_portability.h> +#include <toku_assert.h> +#include <stdio.h> +#include <sys/types.h> +#include <unistd.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> +#include <getopt.h> +#include <signal.h> +#include <memory.h> + +typedef struct { + bool leadingspace; + bool plaintext; + bool header; + bool footer; + bool is_private; + bool recovery_and_txn; + char* progname; + char* homedir; + char* database; + char* subdatabase; + int exitcode; + int recover_flags; + DBTYPE dbtype; + DBTYPE opened_dbtype; + DB* db; + DB_ENV* dbenv; +} dump_globals; + +dump_globals g; + +#define SET_BITS(bitvector, bits) ((bitvector) |= (bits)) +#define REMOVE_BITS(bitvector, bits) ((bitvector) &= ~(bits)) +#define IS_SET_ANY(bitvector, bits) ((bitvector) & (bits)) +#define IS_SET_ALL(bitvector, bits) (((bitvector) & (bits)) == (bits)) + +#define IS_POWER_OF_2(num) ((num) > 0 && ((num) & ((num) - 1)) == 0) + +//DB_ENV->err disabled since it does not use db_strerror +#define PRINT_ERROR(retval, ...) \ +do { \ +if (0) g.dbenv->err(g.dbenv, retval, __VA_ARGS__); \ +else { \ + fprintf(stderr, "\tIn %s:%d %s()\n", __FILE__, __LINE__, __FUNCTION__); \ + fprintf(stderr, "%s: %s:", g.progname, db_strerror(retval)); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + fflush(stderr); \ +} \ +} while (0) + +//DB_ENV->err disabled since it does not use db_strerror, errx does not exist. +#define PRINT_ERRORX(...) \ +do { \ +if (0) g.dbenv->err(g.dbenv, 0, __VA_ARGS__); \ +else { \ + fprintf(stderr, "\tIn %s:%d %s()\n", __FILE__, __LINE__, __FUNCTION__); \ + fprintf(stderr, "%s: ", g.progname); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + fflush(stderr); \ +} \ +} while (0) + +int strtoint32 (char* str, int32_t* num, int32_t min, int32_t max, int base); +int strtouint32 (char* str, uint32_t* num, uint32_t min, uint32_t max, int base); +int strtoint64 (char* str, int64_t* num, int64_t min, int64_t max, int base); +int strtouint64 (char* str, uint64_t* num, uint64_t min, uint64_t max, int base); + +/* + * Convert a string to an integer of type "type". + * + * + * Sets errno and returns: + * EINVAL: str == NULL, num == NULL, or string not of the form [ \t]*[+-]?[0-9]+ + * ERANGE: value out of range specified. (Range of [min, max]) + * + * *num is unchanged on error. + * Returns: + * + */ +#define DEF_STR_TO(name, type, bigtype, strtofunc, frmt) \ +int name(char* str, type* num, type min, type max, int base) \ +{ \ + char* test; \ + bigtype value; \ + \ + assert(str); \ + assert(num); \ + assert(min <= max); \ + assert(g.dbenv || g.progname); \ + assert(base == 0 || (base >= 2 && base <= 36)); \ + \ + errno = 0; \ + while (isspace(*str)) str++; \ + value = strtofunc(str, &test, base); \ + if ((*test != '\0' && *test != '\n') || test == str) { \ + PRINT_ERRORX("%s: Invalid numeric argument\n", str); \ + errno = EINVAL; \ + goto error; \ + } \ + if (errno != 0) { \ + PRINT_ERROR(errno, "%s\n", str); \ + } \ + if (value < min) { \ + PRINT_ERRORX("%s: Less than minimum value (%" frmt ")\n", str, min); \ + goto error; \ + } \ + if (value > max) { \ + PRINT_ERRORX("%s: Greater than maximum value (%" frmt ")\n", str, max); \ + goto error; \ + } \ + *num = value; \ + return EXIT_SUCCESS; \ +error: \ + return errno; \ +} + +DEF_STR_TO(strtoint32, int32_t, int64_t, strtoll, PRId32) +DEF_STR_TO(strtouint32, uint32_t, uint64_t, strtoull, PRIu32) +DEF_STR_TO(strtoint64, int64_t, int64_t, strtoll, PRId64) +DEF_STR_TO(strtouint64, uint64_t, uint64_t, strtoull, PRIu64) + +static inline void +outputbyte(uint8_t ch) +{ + if (g.plaintext) { + if (ch == '\\') printf("\\\\"); + else if (isprint(ch)) printf("%c", ch); + else printf("\\%02x", ch); + } + else printf("%02x", ch); +} + +static inline void +outputstring(char* str) +{ + char* p; + + for (p = str; *p != '\0'; p++) { + outputbyte((uint8_t)*p); + } +} + +static inline void +outputplaintextstring(char* str) +{ + bool old_plaintext = g.plaintext; + g.plaintext = true; + outputstring(str); + g.plaintext = old_plaintext; +} + +static inline int +verify_library_version(void) +{ + int major; + int minor; + + db_version(&major, &minor, NULL); + if (major != DB_VERSION_MAJOR || minor != DB_VERSION_MINOR) { + PRINT_ERRORX("version %d.%d doesn't match library version %d.%d\n", + DB_VERSION_MAJOR, DB_VERSION_MINOR, major, minor); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +static int last_caught = 0; + +static void catch_signal(int which_signal) { + last_caught = which_signal; + if (last_caught == 0) last_caught = SIGINT; +} + +static inline void +init_catch_signals(void) { + signal(SIGINT, catch_signal); + signal(SIGTERM, catch_signal); +#ifdef SIGHUP + signal(SIGHUP, catch_signal); +#endif +#ifdef SIGPIPE + signal(SIGPIPE, catch_signal); +#endif +} + +static inline int +caught_any_signals(void) { + return last_caught != 0; +} + +static inline void +resend_signals(void) { + if (last_caught) { + signal(last_caught, SIG_DFL); + raise(last_caught); + } +} + +static int usage (void); +static int create_init_env(void); +static int dump_database (void); +static int open_database (void); +static int dump_pairs (void); +static int dump_footer (void); +static int dump_header (void); +static int close_database (void); + +int main(int argc, char *const argv[]) { + int ch; + int retval; + + /* Set up the globals. */ + memset(&g, 0, sizeof(g)); + g.leadingspace = true; + //TODO: Uncomment when DB_UNKNOWN + db->get_type are implemented. + g.dbtype = DB_UNKNOWN; + //g.dbtype = DB_BTREE; + g.progname = argv[0]; + g.header = true; + g.footer = true; + g.recovery_and_txn = true; + + if (verify_library_version() != 0) goto error; + + while ((ch = getopt(argc, argv, "d:f:h:klNP:ps:RrVTx")) != EOF) { + switch (ch) { + case ('d'): { + PRINT_ERRORX("-%c option not supported.\n", ch); + goto error; + } + case ('f'): { + if (freopen(optarg, "w", stdout) == NULL) { + fprintf(stderr, + "%s: %s: reopen: %s\n", + g.progname, optarg, strerror(errno)); + goto error; + } + break; + } + case ('h'): { + g.homedir = optarg; + break; + } + case ('k'): { + PRINT_ERRORX("-%c option not supported.\n", ch); + goto error; + } + case ('l'): { + //TODO: Implement (Requires master database support) + PRINT_ERRORX("-%c option not supported.\n", ch); //YET! + goto error; + } + case ('N'): { + PRINT_ERRORX("-%c option not supported.\n", ch); + goto error; + } + case ('P'): { + /* Clear password. */ + memset(optarg, 0, strlen(optarg)); + PRINT_ERRORX("-%c option not supported.\n", ch); + goto error; + } + case ('p'): { + g.plaintext = true; + break; + } + case ('R'): { + //TODO: Uncomment when DB_SALVAGE,DB_AGGRESSIVE are implemented. + /*g.recover_flags |= DB_SALVAGE | DB_AGGRESSIVE;*/ + + //TODO: Implement aggressive recovery (requires db->verify()) + PRINT_ERRORX("-%c option not supported.\n", ch); + goto error; + } + case ('r'): { + //TODO: Uncomment when DB_SALVAGE,DB_AGGRESSIVE are implemented. + /*g.recover_flags |= DB_SALVAGE;*/ + + //TODO: Implement recovery (requires db->verify()) + PRINT_ERRORX("-%c option not supported.\n", ch); + goto error; + } + case ('s'): { + g.subdatabase = optarg; + break; + } + case ('V'): { + printf("%s\n", db_version(NULL, NULL, NULL)); + goto cleanup; + } + case ('T'): { + g.plaintext = true; + g.leadingspace = false; + g.header = false; + g.footer = false; + break; + } + case ('x'): { + g.recovery_and_txn = false; + break; + } + case ('?'): + default: { + g.exitcode = usage(); + goto cleanup; + } + } + } + argc -= optind; + argv += optind; + + //TODO: Uncomment when DB_SALVAGE,DB_AGGRESSIVE,DB_PRINTABLE,db->verify are implemented. + /* + if (g.plaintext) g.recover_flags |= DB_PRINTABLE; + + if (g.subdatabase != NULL && IS_SET_ALL(g.recover_flags, DB_SALVAGE)) { + if (IS_SET_ALL(g.recover_flags, DB_AGGRESSIVE)) { + PRINT_ERRORX("The -s and -R options may not both be specified.\n"); + goto error; + } + PRINT_ERRORX("The -s and -r options may not both be specified.\n"); + goto error; + + } + */ + + if (argc != 1) { + g.exitcode = usage(); + goto cleanup; + } + + init_catch_signals(); + + g.database = argv[0]; + if (caught_any_signals()) goto cleanup; + if (create_init_env() != 0) goto error; + if (caught_any_signals()) goto cleanup; + if (dump_database() != 0) goto error; + if (false) { +error: + g.exitcode = EXIT_FAILURE; + fprintf(stderr, "%s: Quitting out due to errors.\n", g.progname); + } +cleanup: + if (g.dbenv && (retval = g.dbenv->close(g.dbenv, 0)) != 0) { + g.exitcode = EXIT_FAILURE; + fprintf(stderr, "%s: %s: dbenv->close\n", g.progname, db_strerror(retval)); + } + // if (g.subdatabase) free(g.subdatabase); + resend_signals(); + + return g.exitcode; +} + +int dump_database() +{ + int retval; + + /* Create a database handle. */ + retval = db_create(&g.db, g.dbenv, 0); + if (retval != 0) { + PRINT_ERROR(retval, "db_create"); + return EXIT_FAILURE; + } + + /* + TODO: If/when supporting encryption + if (g.password && (retval = db->set_flags(db, DB_ENCRYPT))) { + PRINT_ERROR(ret, "DB->set_flags: DB_ENCRYPT"); + goto error; + } + */ + if (open_database() != 0) goto error; + if (caught_any_signals()) goto cleanup; + if (g.header && dump_header() != 0) goto error; + if (caught_any_signals()) goto cleanup; + if (dump_pairs() != 0) goto error; + if (caught_any_signals()) goto cleanup; + if (g.footer && dump_footer() != 0) goto error; + + if (false) { +error: + g.exitcode = EXIT_FAILURE; + } +cleanup: + + if (close_database() != 0) g.exitcode = EXIT_FAILURE; + + return g.exitcode; +} + +int usage() +{ + fprintf(stderr, + "usage: %s [-pVT] [-x] [-f output] [-h home] [-s database] db_file\n", + g.progname); + return EXIT_FAILURE; +} + +int create_init_env() +{ + int retval; + DB_ENV* dbenv; + int flags; + //TODO: Experiments to determine right cache size for tokudb, or maybe command line argument. + + retval = db_env_create(&dbenv, 0); + if (retval) { + fprintf(stderr, "%s: db_dbenv_create: %s\n", g.progname, db_strerror(retval)); + goto error; + } + ///TODO: UNCOMMENT/IMPLEMENT dbenv->set_errfile(dbenv, stderr); + dbenv->set_errpfx(dbenv, g.progname); + /* + TODO: Anything for encryption? + */ + + /* Open the dbenvironment. */ + g.is_private = false; + //flags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_USE_ENVIRON; + flags = DB_INIT_LOCK | DB_INIT_MPOOL; ///TODO: UNCOMMENT/IMPLEMENT | DB_USE_ENVIRON; + if (g.recovery_and_txn) { + SET_BITS(flags, DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER); + } + + /* + ///TODO: UNCOMMENT/IMPLEMENT Notes: We require DB_PRIVATE + if (!dbenv->open(dbenv, g.homedir, flags, 0)) goto success; + */ + + /* + ///TODO: UNCOMMENT/IMPLEMENT + retval = dbenv->set_cachesize(dbenv, 0, cache, 1); + if (retval) { + PRINT_ERROR(retval, "DB_ENV->set_cachesize"); + goto error; + } + */ + g.is_private = true; + //TODO: Do we want to support transactions even in single-process mode? + //Logging is not necessary.. this is read-only. + //However, do we need to use DB_INIT_LOG to join a logging environment? + //REMOVE_BITS(flags, DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN); + SET_BITS(flags, DB_CREATE | DB_PRIVATE); + + retval = dbenv->open(dbenv, g.homedir, flags, 0); + if (retval) { + PRINT_ERROR(retval, "DB_ENV->open"); + goto error; + } + g.dbenv = dbenv; + return EXIT_SUCCESS; + +error: + return EXIT_FAILURE; +} + +#define DUMP_FLAG(bit, dump) if (IS_SET_ALL(flags, bit)) printf(dump); + +#define DUMP_IGNORED_FLAG(bit, dump) + + +int dump_header() +{ + uint32_t flags; + int retval; + DB* db = g.db; + + assert(g.header); + printf("VERSION=3\n"); + printf("format=%s\n", g.plaintext ? "print" : "bytevalue"); + //TODO: Uncomment when DB_UNKNOWN + db->get_type are implemented. + /*assert(g.dbtype == DB_BTREE || (g.dbtype == DB_UNKNOWN && g.opened_dbtype == DB_BTREE));*/ + printf("type=btree\n"); + //TODO: Get page size from db. Currently tokudb does not support db->get_pagesize. + //Don't print this out //printf("db_pagesize=4096\n"); + if (g.subdatabase) { + printf("subdatabase="); + outputplaintextstring(g.subdatabase); + printf("\n"); + } + //TODO: Uncomment when db->get_flags is implemented + if ((retval = db->get_flags(db, &flags)) != 0) { + PRINT_ERROR(retval, "DB->get_flags"); + goto error; + } + DUMP_IGNORED_FLAG(DB_CHKSUM, "chksum=1\n"); + DUMP_IGNORED_FLAG(DB_RECNUM, "recnum=1\n"); + printf("HEADER=END\n"); + + if (ferror(stdout)) goto error; + return EXIT_SUCCESS; + +error: + return EXIT_FAILURE; +} + +int dump_footer() +{ + printf("DATA=END\n"); + if (ferror(stdout)) goto error; + + return EXIT_SUCCESS; +error: + return EXIT_FAILURE; +} + +int open_database() +{ + DB* db = g.db; + int retval; + + int open_flags = 0;//|DB_RDONLY; + //TODO: Transaction auto commit stuff + SET_BITS(open_flags, DB_AUTO_COMMIT); + + retval = db->open(db, NULL, g.database, g.subdatabase, g.dbtype, open_flags, 0666); + if (retval != 0) { + PRINT_ERROR(retval, "DB->open: %s", g.database); + goto error; + } + //TODO: Uncomment when DB_UNKNOWN + db->get_type are implemented. + /* + retval = db->get_type(db, &g.opened_dbtype); + if (retval != 0) { + PRINT_ERROR(retval, "DB->get_type"); + goto error; + } + if (g.opened_dbtype != DB_BTREE) { + PRINT_ERRORX("Unsupported db type %d\n", g.opened_dbtype); + goto error; + } + if (g.dbtype != DB_UNKNOWN && g.opened_dbtype != g.dbtype) { + PRINT_ERRORX("DBTYPE %d does not match opened DBTYPE %d.\n", g.dbtype, g.opened_dbtype); + goto error; + }*/ + return EXIT_SUCCESS; +error: + fprintf(stderr, "Quitting out due to errors.\n"); + return EXIT_FAILURE; +} + +static int dump_dbt(DBT* dbt) +{ + char* str; + uint32_t idx; + + assert(dbt); + str = (char*)dbt->data; + if (g.leadingspace) printf(" "); + if (dbt->size > 0) { + assert(dbt->data); + for (idx = 0; idx < dbt->size; idx++) { + outputbyte(str[idx]); + if (ferror(stdout)) { + perror("stdout"); + goto error; + } + } + } + printf("\n"); + if (false) { +error: + g.exitcode = EXIT_FAILURE; + } + return g.exitcode; +} + +int dump_pairs() +{ + int retval; + DBT key; + DBT data; + DB* db = g.db; + DBC* dbc = NULL; + + memset(&key, 0, sizeof(key)); + memset(&data, 0, sizeof(data)); + + DB_TXN* txn = NULL; + if (g.recovery_and_txn) { + retval = g.dbenv->txn_begin(g.dbenv, NULL, &txn, 0); + if (retval) { + PRINT_ERROR(retval, "DB_ENV->txn_begin"); + goto error; + } + } + + if ((retval = db->cursor(db, txn, &dbc, 0)) != 0) { + PRINT_ERROR(retval, "DB->cursor"); + goto error; + } + while ((retval = dbc->c_get(dbc, &key, &data, DB_NEXT)) == 0) { + if (caught_any_signals()) goto cleanup; + if (dump_dbt(&key) != 0) goto error; + if (dump_dbt(&data) != 0) goto error; + } + if (retval != DB_NOTFOUND) { + PRINT_ERROR(retval, "DBC->c_get"); + goto error; + } + + + if (false) { +error: + g.exitcode = EXIT_FAILURE; + } +cleanup: + if (dbc && (retval = dbc->c_close(dbc)) != 0) { + PRINT_ERROR(retval, "DBC->c_close"); + g.exitcode = EXIT_FAILURE; + } + if (txn) { + if (retval) { + int r2 = txn->abort(txn); + if (r2) PRINT_ERROR(r2, "DB_TXN->abort"); + } + else { + retval = txn->commit(txn, 0); + if (retval) PRINT_ERROR(retval, "DB_TXN->abort"); + } + } + return g.exitcode; +} + +int close_database() +{ + DB* db = g.db; + int retval; + + assert(db); + if ((retval = db->close(db, 0)) != 0) { + PRINT_ERROR(retval, "DB->close"); + goto error; + } + return EXIT_SUCCESS; +error: + return EXIT_FAILURE; +} diff --git a/storage/tokudb/PerconaFT/tools/tokuft_logprint.cc b/storage/tokudb/PerconaFT/tools/tokuft_logprint.cc new file mode 100644 index 00000000..924eee2d --- /dev/null +++ b/storage/tokudb/PerconaFT/tools/tokuft_logprint.cc @@ -0,0 +1,74 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +/* Dump the log from stdin to stdout. */ +#include "ft/ft.h" +#include "ft/log_header.h" +#include "ft/logger/logger.h" + +using namespace std; + +int main (int argc, const char *const argv[]) { + int r = toku_ft_layer_init(); + assert_zero(r); + + int count=-1; + while (argc>1) { + if (strcmp(argv[1], "--oldcode")==0) { + fprintf(stderr,"Old code no longer works.\n"); + exit(1); + } else { + count = atoi(argv[1]); + } + argc--; argv++; + } + int i; + uint32_t version; + r = toku_read_and_print_logmagic(stdin, &version); + for (i=0; i!=count; i++) { + r = toku_logprint_one_record(stdout, stdin); + if (r==EOF) break; + if (r!=0) { + fflush(stdout); + fprintf(stderr, "Problem in log err=%d\n", r); + exit(1); + } + } + toku_ft_layer_destroy(); + return 0; +} diff --git a/storage/tokudb/PerconaFT/tools/tokuftdump.cc b/storage/tokudb/PerconaFT/tools/tokuftdump.cc new file mode 100644 index 00000000..44edb151 --- /dev/null +++ b/storage/tokudb/PerconaFT/tools/tokuftdump.cc @@ -0,0 +1,1246 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +// Dump a fractal tree file + +#include <ctype.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <inttypes.h> +#include <limits.h> +#include <string> +#include <iostream> +#include <fstream> +#include <map> +#include <string> +#include <string.h> +#include "ft/serialize/block_table.h" +#include "ft/cachetable/cachetable.h" +#include "ft/ft.h" +#include "ft/ft-internal.h" +#include "ft/serialize/ft-serialize.h" +#include "ft/serialize/ft_node-serialize.h" +#include "ft/node.h" + +using namespace std; + +static int do_dump_data = 1; +static int do_interactive = 0; +static int do_json = 0; +static int do_header = 0; +static int do_fragmentation = 0; +static int do_garbage = 0; +static int do_translation_table = 0; +static int do_summary = 0; +static int do_rootnode = 0; +static int do_node = 0; +static BLOCKNUM do_node_num; +static int do_tsv = 0; +static const char *arg0; +static const char *fname; + +//it holdes the messges count for each FT's node +typedef struct nodeMessage{ + int id; + int clean;//0=clean >=1 dirty + int *count;//holds the messages + nodeMessage *nextNode; +}NMC; +enum { maxline = 128}; + +static int printNodeMessagesToSTDout(NMC* ptr); + +static int printLevelSTDout(int *); + +static void treeToSTDout(NMC *msgs[], int height); + +static void format_time(const uint64_t time_int, char *buf) { + time_t timer = (time_t) time_int; + ctime_r(&timer, buf); + assert(buf[24] == '\n'); + buf[24] = 0; +} + +static void print_item(const void *val, uint32_t len) { + printf("\""); + uint32_t i; + for (i=0; i<len; i++) { + unsigned char ch = ((unsigned char*)val)[i]; + if (isprint(ch) && ch!='\\' && ch!='"') { + printf("%c", ch); + } else { + printf("\\%03o", ch); + } + } + printf("\""); +} + +static void simple_hex_dump(unsigned char *vp, uint64_t size) { + for (uint64_t i = 0; i < size; i++) { + unsigned char c = vp[i]; + printf("%2.2X", c); + } +} + +static void hex_dump(unsigned char *vp, uint64_t offset, uint64_t size) { + uint64_t n = size / 32; + for (uint64_t i = 0; i < n; i++) { + printf("%" PRIu64 ": ", offset); + for (uint64_t j = 0; j < 32; j++) { + unsigned char c = vp[j]; + printf("%2.2X", c); + if (((j+1) % 4) == 0) + printf(" "); + } + for (uint64_t j = 0; j < 32; j++) { + unsigned char c = vp[j]; + printf("%c", isprint(c) ? c : ' '); + } + printf("\n"); + vp += 32; + offset += 32; + } + size = size % 32; + for (uint64_t i=0; i<size; i++) { + if ((i % 32) == 0) + printf("%" PRIu64 ": ", offset+i); + printf("%2.2X", vp[i]); + if (((i+1) % 4) == 0) + printf(" "); + if (((i+1) % 32) == 0) + printf("\n"); + } + printf("\n"); +} + +static void dump_descriptor(DESCRIPTOR d) { + printf(" descriptor size %u ", d->dbt.size); + simple_hex_dump((unsigned char*) d->dbt.data, d->dbt.size); + printf("\n"); +} + +static void open_header(int fd, FT *header, CACHEFILE cf) { + FT ft = NULL; + int r; + const char *fn = toku_cachefile_fname_in_env(cf); + r = toku_deserialize_ft_from (fd, fn, MAX_LSN, &ft); + if (r != 0) { + fprintf(stderr, "%s: can not deserialize from %s error %d\n", arg0, fname, r); + exit(1); + } + assert_zero(r); + ft->cf = cf; + *header = ft; +} + +static void dump_header(FT ft) { + char timestr[26]; + printf("ft:\n"); + printf(" layout_version=%d\n", ft->h->layout_version); + printf(" layout_version_original=%d\n", ft->h->layout_version_original); + printf(" layout_version_read_from_disk=%d\n", ft->layout_version_read_from_disk); + printf(" build_id=%d\n", ft->h->build_id); + printf(" build_id_original=%d\n", ft->h->build_id_original); + format_time(ft->h->time_of_creation, timestr); + printf(" time_of_creation= %" PRIu64 " %s\n", ft->h->time_of_creation, timestr); + format_time(ft->h->time_of_last_modification, timestr); + printf(" time_of_last_modification=%" PRIu64 " %s\n", ft->h->time_of_last_modification, timestr); + printf(" dirty=%d\n", ft->h->dirty()); + printf(" checkpoint_count=%" PRId64 "\n", ft->h->checkpoint_count); + printf(" checkpoint_lsn=%" PRId64 "\n", ft->h->checkpoint_lsn.lsn); + printf(" nodesize=%u\n", ft->h->nodesize); + printf(" fanout=%u\n", ft->h->fanout); + printf(" basementnodesize=%u\n", ft->h->basementnodesize); + printf(" compression_method=%u\n", (unsigned) ft->h->compression_method); + printf(" unnamed_root=%" PRId64 "\n", ft->h->root_blocknum.b); + printf(" flags=%u\n", ft->h->flags); + dump_descriptor(&ft->descriptor); + printf(" estimated numrows=%" PRId64 "\n", ft->in_memory_stats.numrows); + printf(" estimated numbytes=%" PRId64 "\n", ft->in_memory_stats.numbytes); + printf(" logical row count=%" PRId64 "\n", ft->in_memory_logical_rows); +} + +static int64_t getRootNode(FT ft) { + return ft->h->root_blocknum.b; +} + +static int print_le(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t idx UU(), void *const ai UU()) { + unsigned int *le_index = (unsigned int *) ai; + printf("%u: ", *le_index); *le_index += 1; + print_klpair(stdout, key, keylen, le); + printf("\n"); + return 0; +} + +static int getHeight(int fd, BLOCKNUM blocknum, FT ft){ + FTNODE n; + FTNODE_DISK_DATA ndd = nullptr; + ftnode_fetch_extra bfe; + bfe.create_for_full_read(ft); + int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe); + assert_zero(r); + assert(n!=0); + return n->height; +} + +static FTNODE getNode(int fd, BLOCKNUM blocknum, FT ft) { + FTNODE n; + FTNODE_DISK_DATA ndd = nullptr; + ftnode_fetch_extra bfe; + bfe.create_for_full_read(ft); + int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe); + assert_zero(r);; + return n; +} + +static int countNodes(NMC *level){ + int count=0; + NMC *ptr=level; + while(ptr!=NULL){ + count++; + ptr=ptr->nextNode; + } + return count; +} + +static int * countMessages(NMC *level){ + int *counts=new int[16]; + for(int i=0;i<16;i++){ + counts[i]=0; + } + NMC *ptr=level; + while(ptr!=NULL){ + for(int i=0;i<16;i++){ + counts[i]+=ptr->count[i]; + } + ptr=ptr->nextNode; + } + return counts; +} + +static NMC * getLast(NMC *level){ + if (level==NULL) return NULL; + NMC *ptr=level; + while(ptr->nextNode!=NULL){ + ptr=ptr->nextNode; + } + return ptr; +} + +/* + * Prints the total messages at each to STDout + */ +static int printLevelSTDout(int *count){ + int isEmpty=0; + for(int j=0;j<16;j++){ + if(count[j]>0){ + cout <<count[j]<<" "; + isEmpty++; + switch (j) { + case FT_INSERT: cout <<"INSERT(s) "; break; + case FT_INSERT_NO_OVERWRITE: cout <<"INSERT_NO_OVERWRITE(s) "; break; + case FT_DELETE_ANY: cout <<"DELETE_ANY(s) "; break; + case FT_ABORT_ANY: cout <<"ABORT_ANY(s) "; break; + case FT_COMMIT_ANY: cout <<"COMMIT_ANY(s) "; break; + case FT_COMMIT_BROADCAST_ALL: cout <<"COMMIT_BROADCAST_ALL(s) "; break; + case FT_COMMIT_BROADCAST_TXN: cout <<"COMMIT_BROADCAST_TXN(s) "; break; + case FT_ABORT_BROADCAST_TXN: cout <<"ABORT_BROADCAST_TXN(s) "; break; + case FT_OPTIMIZE: cout <<"OPTIMIZE(s) "; break; + case FT_OPTIMIZE_FOR_UPGRADE: cout <<"OPTIMIZE_FOR_UPGRADE(s) "; break; + case FT_UPDATE: cout <<"UPDATE(s) "; break; + case FT_UPDATE_BROADCAST_ALL: cout <<"UPDATE_BROADCAST_ALL(s) "; break; + } + + } + } + return isEmpty; +} + +/* + * Prints the total # of messages in a node to STD output + */ +static int printNodeMessagesToSTDout(NMC *ptr){ + cout <<"\nNode :"<<ptr->id<<" has :"; + for(int j=0;j<16;j++){ + if(ptr->count[j]>0){ + cout <<ptr->count[j]<<" "; + switch (j) { + case FT_INSERT: cout <<"INSERT(s) "; break; + case FT_INSERT_NO_OVERWRITE: cout <<"INSERT_NO_OVERWRITE(s) "; break; + case FT_DELETE_ANY: cout <<"DELETE_ANY(s) "; break; + case FT_ABORT_ANY: cout <<"ABORT_ANY(s) "; break; + case FT_COMMIT_ANY: cout <<"COMMIT_ANY(s) "; break; + case FT_COMMIT_BROADCAST_ALL: cout <<"COMMIT_BROADCAST_ALL(s) "; break; + case FT_COMMIT_BROADCAST_TXN: cout <<"COMMIT_BROADCAST_TXN(s) "; break; + case FT_ABORT_BROADCAST_TXN: cout <<"ABORT_BROADCAST_TXN(s) "; break; + case FT_OPTIMIZE: cout <<"OPTIMIZE(s) "; break; + case FT_OPTIMIZE_FOR_UPGRADE: cout <<"OPTIMIZE_FOR_UPGRADE(s) "; break; + case FT_UPDATE: cout <<"UPDATE(s) "; break; + case FT_UPDATE_BROADCAST_ALL: cout <<"UPDATE_BROADCAST_ALL(s) "; break; + } + } + } + return 1; +} + +static void levelToSTDout(NMC *list, int level){ + NMC *ptr=list; + cout <<endl<<"Height : "<<level<<endl; + while(ptr!=NULL){ + if(ptr->clean!=0){ + printNodeMessagesToSTDout(ptr); + } + else{ + cout << "\nNode : "<<ptr->id<<" has no messages"; + } + ptr=ptr->nextNode; + } + cout <<endl; +} + +/* + * prints the tree total # of nodes and total # of messages at each height in : + * STDout in human readable format + */ +static void treeToSTDout(NMC *msgs[], int height){ + for(int i=height; i>=0 ; i--){ + cout <<"At height "<<i; + int *counts=countMessages(msgs[i]); + cout <<"\n Node Count: "<< countNodes(msgs[i])<<endl; + cout <<" Messages: "; + if(printLevelSTDout(counts)==0) cout <<"0\n"; + else cout <<endl; + } +} + +//traverse through the FT and report back the count of messages in every node +static void countMessagesInFT(int fd, BLOCKNUM blocknum, FT ft,NMC *msgs[]){ + FTNODE n=getNode(fd,blocknum,ft); + + NMC *last=NULL; + if(msgs[n->height]==NULL){ + last = msgs[n->height]=new NMC; + }else { + last=getLast(msgs[n->height]); + last->nextNode=new NMC; + last=last->nextNode; + } + last->id=blocknum.b; + last->count=new int[16]; + for(int i=0;i<16;i++){ + last->count[i]=0; + } + last->clean=0; + last->nextNode=NULL; + + if (n->height==0){ + toku_ftnode_free(&n); + return; + } + for(int i=0;i<n->n_children;i++){ + NONLEAF_CHILDINFO bnc = BNC(n, i); + if (n->height==1 && n->bp[i].ptr.tag==BCT_NULL){ + cout <<n->bp[i].ptr.tag; + } + auto dump_fn=[&](const ft_msg &msg, bool UU(is_fresh)) { + enum ft_msg_type type = (enum ft_msg_type) msg.type(); + last->count[type]++; + last->clean=1; + return 0; + }; + + bnc->msg_buffer.iterate(dump_fn); + + blocknum=make_blocknum(BP_BLOCKNUM(n, i).b); + countMessagesInFT(fd,blocknum,ft, msgs); + } + + toku_ftnode_free(&n); +} + +static void dump_node(int fd, BLOCKNUM blocknum, FT ft) { + FTNODE n; + FTNODE_DISK_DATA ndd = nullptr; + ftnode_fetch_extra bfe; + bfe.create_for_full_read(ft); + int r = toku_deserialize_ftnode_from (fd, blocknum, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe); + assert_zero(r); + assert(n!=0); + printf("ftnode\n"); + DISKOFF disksize, diskoffset; + ft->blocktable.translate_blocknum_to_offset_size(blocknum, &diskoffset, &disksize); + printf(" diskoffset =%" PRId64 "\n", diskoffset); + printf(" disksize =%" PRId64 "\n", disksize); + printf(" serialize_size =%u\n", toku_serialize_ftnode_size(n)); + printf(" flags =%u\n", n->flags); + printf(" blocknum=%" PRId64 "\n", n->blocknum.b); + //printf(" log_lsn =%lld\n", n->log_lsn.lsn); // The log_lsn is a memory-only value. + printf(" height =%d\n", n->height); + printf(" layout_version=%d\n", n->layout_version); + printf(" layout_version_original=%d\n", n->layout_version_original); + printf(" layout_version_read_from_disk=%d\n", n->layout_version_read_from_disk); + printf(" build_id=%d\n", n->build_id); + printf(" max_msn_applied_to_node_on_disk=%" PRId64 " (0x%" PRIx64 ")\n", n->max_msn_applied_to_node_on_disk.msn, n->max_msn_applied_to_node_on_disk.msn); + printf(" io time %lf decompress time %lf deserialize time %lf\n", + tokutime_to_seconds(bfe.io_time), + tokutime_to_seconds(bfe.decompress_time), + tokutime_to_seconds(bfe.deserialize_time)); + + printf(" n_children=%d\n", n->n_children); + printf(" pivotkeys.total_size()=%u\n", (unsigned) n->pivotkeys.total_size()); + + if (n->height > 0) { + printf(" pivots:\n"); + } else { + printf("LEAF keys:\n"); + } + + for (int i=0; i<n->n_children-1; i++) { + const DBT piv = n->pivotkeys.get_pivot(i); + printf(" pivot %2d:", i); + if (n->flags) + printf(" flags=%x ", n->flags); + print_item(piv.data, piv.size); + printf("\n"); + } + + if (n->height > 0) { + printf(" children:\n"); + } else { + printf("LEAF data:\n"); + } + + for (int i=0; i<n->n_children; i++) { + printf(" child %d: ", i); + if (n->height > 0) { + printf("%" PRId64 "\n", BP_BLOCKNUM(n, i).b); + NONLEAF_CHILDINFO bnc = BNC(n, i); + unsigned int n_bytes = toku_bnc_nbytesinbuf(bnc); + int n_entries = toku_bnc_n_entries(bnc); + if (n_bytes > 0 || n_entries > 0) { + printf(" buffer contains %u bytes (%d items)\n", n_bytes, n_entries); + } + if (do_dump_data) { + struct dump_data_fn { + int operator()(const ft_msg &msg, bool UU(is_fresh)) { + enum ft_msg_type type = (enum ft_msg_type) msg.type(); + MSN msn = msg.msn(); + XIDS xids = msg.xids(); + const void *key = msg.kdbt()->data; + const void *data = msg.vdbt()->data; + uint32_t keylen = msg.kdbt()->size; + uint32_t datalen = msg.vdbt()->size; + printf(" msn=%" PRIu64 " (0x%" PRIx64 ") ", msn.msn, msn.msn); + printf(" TYPE="); + switch (type) { + case FT_NONE: printf("NONE"); goto ok; + case FT_INSERT: printf("INSERT"); goto ok; + case FT_INSERT_NO_OVERWRITE: printf("INSERT_NO_OVERWRITE"); goto ok; + case FT_DELETE_ANY: printf("DELETE_ANY"); goto ok; + case FT_ABORT_ANY: printf("ABORT_ANY"); goto ok; + case FT_COMMIT_ANY: printf("COMMIT_ANY"); goto ok; + case FT_COMMIT_BROADCAST_ALL: printf("COMMIT_BROADCAST_ALL"); goto ok; + case FT_COMMIT_BROADCAST_TXN: printf("COMMIT_BROADCAST_TXN"); goto ok; + case FT_ABORT_BROADCAST_TXN: printf("ABORT_BROADCAST_TXN"); goto ok; + case FT_OPTIMIZE: printf("OPTIMIZE"); goto ok; + case FT_OPTIMIZE_FOR_UPGRADE: printf("OPTIMIZE_FOR_UPGRADE"); goto ok; + case FT_UPDATE: printf("UPDATE"); goto ok; + case FT_UPDATE_BROADCAST_ALL: printf("UPDATE_BROADCAST_ALL"); goto ok; + } + printf("HUH?"); +ok: + printf(" xid="); + toku_xids_fprintf(stdout, xids); + printf(" "); + print_item(key, keylen); + if (datalen>0) { + printf(" "); + print_item(data, datalen); + } + printf("\n"); + return 0; + } + } dump_fn; + bnc->msg_buffer.iterate(dump_fn); + } + } else { + printf(" n_bytes_in_buffer= %" PRIu64 "", BLB_DATA(n, i)->get_disk_size()); + printf(" items_in_buffer=%u\n", BLB_DATA(n, i)->num_klpairs()); + if (do_dump_data) { + unsigned int le_index = 0; + BLB_DATA(n, i)->iterate<void, print_le>(&le_index); + } + } + } + toku_ftnode_free(&n); + toku_free(ndd); +} + +static void dump_block_translation(FT ft, uint64_t offset) { + ft->blocktable.blocknum_dump_translation(make_blocknum(offset)); +} + +static void dump_fragmentation(int UU(f), FT ft, int tsv) { + int64_t used_space; + int64_t total_space; + ft->blocktable.internal_fragmentation(&total_space, &used_space); + int64_t fragsizes = total_space - used_space; + + if (tsv) { + printf("%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%.1f\n", used_space, total_space, fragsizes, + 100. * ((double)fragsizes / (double)(total_space))); + } else { + printf("used_size\t%" PRId64 "\n", used_space); + printf("total_size\t%" PRId64 "\n", total_space); + printf("fragsizes\t%" PRId64 "\n", fragsizes); + printf("fragmentation\t%.1f\n", 100. * ((double)fragsizes / (double)(total_space))); + } +} + +typedef struct { + int fd; + FT ft; + uint64_t blocksizes; + uint64_t leafsizes; + uint64_t leafblocks; +} frag_help_extra; + +static int nodesizes_helper(BLOCKNUM b, int64_t size, int64_t UU(address), void *extra) { + frag_help_extra *CAST_FROM_VOIDP(info, extra); + FTNODE n; + FTNODE_DISK_DATA ndd = NULL; + ftnode_fetch_extra bfe; + bfe.create_for_full_read(info->ft); + int r = toku_deserialize_ftnode_from(info->fd, b, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe); + if (r==0) { + info->blocksizes += size; + if (n->height == 0) { + info->leafsizes += size; + info->leafblocks++; + } + toku_ftnode_free(&n); + toku_free(ndd); + } + return 0; +} + +static void dump_nodesizes(int fd, FT ft) { + frag_help_extra info; + memset(&info, 0, sizeof(info)); + info.fd = fd; + info.ft = ft; + ft->blocktable.iterate(block_table::TRANSLATION_CHECKPOINTED, + nodesizes_helper, &info, true, true); + printf("leafblocks\t%" PRIu64 "\n", info.leafblocks); + printf("blocksizes\t%" PRIu64 "\n", info.blocksizes); + printf("leafsizes\t%" PRIu64 "\n", info.leafsizes); +} + +/* ===== struct and function to get a summary of atree ===== */ + +typedef struct { + int fd; + FT ft; + uint64_t blocksizes; + uint64_t leafsizes; + uint64_t serialsize; // sizes of serialized data (assume uncomressed) + uint64_t leafblocks; // count of leaf nodes + uint64_t nonleafnode_cnt; // count of non-leaf nodes + uint64_t maxheight; // height of the tree + uint64_t msg_cnt; // message count in non-leafs + uint64_t msg_size; // size (in bytes of all messages in non-leafs + uint64_t pairs_cnt; // count of pairs in leaf nodes + std::map<int, int> height_cnt; // count of nodes per height + std::map<int, int> hmsg_cnt; // count of message per height + std::map<int, uint64_t> hmsg_size; // size of message per height + std::map<int, uint64_t> hdisk_size; // disk size per height + std::map<int, uint64_t> hserial_size; // serial size per height +} summary_help_extra; + +static int summary_helper(BLOCKNUM b, int64_t size, int64_t UU(address), void *extra) { + summary_help_extra *CAST_FROM_VOIDP(info, extra); + FTNODE n; + FTNODE_DISK_DATA ndd = NULL; + ftnode_fetch_extra bfe; + + bfe.create_for_full_read(info->ft); + int r = toku_deserialize_ftnode_from(info->fd, b, 0 /*pass zero for hash, it doesn't matter*/, &n, &ndd, &bfe); + if (r==0) { + info->blocksizes += size; + + (info->height_cnt)[n->height]++; + + if (n->height == 0) { + info->leafsizes += size; + info->leafblocks++; + } else { + info->nonleafnode_cnt++; + } + + info->hdisk_size[n->height] += size; + auto serialsize = toku_serialize_ftnode_size(n); + info->serialsize += serialsize; + info->hserial_size[n->height] += serialsize; + + + if ((uint64_t)n->height > info->maxheight) { + info->maxheight = n->height; + } + + + + for (int i=0; i<n->n_children; i++) { + //printf(" child %d: ", i); + if (n->height > 0) { + NONLEAF_CHILDINFO bnc = BNC(n, i); + unsigned int n_bytes = toku_bnc_nbytesinbuf(bnc); + int n_entries = toku_bnc_n_entries(bnc); + //if (n_bytes > 0 || n_entries > 0) { + // printf(" buffer contains %u bytes (%d items)\n", n_bytes, n_entries); + //} + info->msg_cnt += n_entries; + info->msg_size += n_bytes; + info->hmsg_cnt[n->height] += n_entries; + info->hmsg_size[n->height] += n_bytes; + } else { + info->pairs_cnt += BLB_DATA(n, i)->num_klpairs(); + } + } + if (n->height ==0) { + info->hmsg_cnt[0] += n->n_children; // this way we count partitions per leaf node + } + + + toku_ftnode_free(&n); + toku_free(ndd); + } + return 0; +} + +static std::string humanNumber(uint64_t value) { + std::string numWithCommas = to_string(value); + int insertPosition = numWithCommas.length() - 3; + while (insertPosition > 0) { + numWithCommas.insert(insertPosition, ","); + insertPosition-=3; + } + return numWithCommas; +} + +static void dump_summary(int fd, FT ft) { + summary_help_extra info; + //memset(&info, 0, sizeof(info)); + info.fd = fd; + info.ft = ft; + info.blocksizes = 0; + info.leafsizes = 0; + info.serialsize = 0; + info.leafblocks = 0; + info.nonleafnode_cnt = 0; + info.maxheight = 0; + info.msg_cnt = 0; + info.msg_size = 0; + info.pairs_cnt = 0; + + ft->blocktable.iterate(block_table::TRANSLATION_CHECKPOINTED, + summary_helper, &info, true, true); + printf("leaf nodes:\t%" PRIu64 "\n", info.leafblocks); + printf("non-leaf nodes:\t%" PRIu64 "\n", info.nonleafnode_cnt); + printf("Leaf size:\t%s\n", humanNumber(info.leafsizes).c_str()); + printf("Total size:\t%s\n", humanNumber(info.blocksizes).c_str()); + printf("Total uncompressed size:\t%s\n", humanNumber(info.serialsize).c_str()); + printf("Messages count:\t%" PRIu64 "\n", info.msg_cnt); + printf("Messages size:\t%s\n", humanNumber(info.msg_size).c_str()); + printf("Records count:\t%" PRIu64 "\n", info.pairs_cnt); + printf("Tree height:\t%" PRIu64 "\n", info.maxheight); + for(auto elem : info.height_cnt) { + std::string hdr; + double children_per_node; + if (elem.first == 0) { + hdr = "basement nodes"; + children_per_node = (double)info.hmsg_cnt[0]/elem.second; + } else { + hdr = "msg cnt"; + children_per_node = (double)info.height_cnt[elem.first-1]/elem.second; + } + + printf("height: %d, nodes count: %d; avg children/node: %f\n\t %s: %d; msg size: %s; disksize: %s; uncompressed size: %s; ratio: %f\n", + elem.first, elem.second, children_per_node, + hdr.c_str(), + info.hmsg_cnt[elem.first], + humanNumber(info.hmsg_size[elem.first]).c_str(), + humanNumber(info.hdisk_size[elem.first]).c_str(), + humanNumber(info.hserial_size[elem.first]).c_str(), + (double)info.hserial_size[elem.first]/info.hdisk_size[elem.first] ); + } +} + +/* ===== end of summary ===== */ + +static void dump_garbage_stats(int fd, FT ft) { + assert(fd == toku_cachefile_get_fd(ft->cf)); + uint64_t total_space = 0; + uint64_t used_space = 0; + toku_ft_get_garbage(ft, &total_space, &used_space); + printf("garbage total size :%20" PRIu64 "\n", total_space); + printf("garbage used size :%20" PRIu64 "\n", used_space); + float a=used_space,b=total_space; + + float percentage=((1-a/b)*100); + printf("Total garbage : %2.3f%%\n", percentage); +} + +typedef struct __dump_node_extra { + int fd; + FT ft; +} dump_node_extra; + +static int dump_node_wrapper(BLOCKNUM b, int64_t UU(size), int64_t UU(address), void *extra) { + dump_node_extra *CAST_FROM_VOIDP(info, extra); + dump_node(info->fd, b, info->ft); + return 0; +} + +static uint32_t get_unaligned_uint32(unsigned char *p) { + uint32_t n; + memcpy(&n, p, sizeof n); + return n; +} + +struct dump_sub_block { + uint32_t compressed_size; + uint32_t uncompressed_size; + uint32_t xsum; +}; + +static void sub_block_deserialize(struct dump_sub_block *sb, unsigned char *sub_block_header) { + sb->compressed_size = toku_dtoh32(get_unaligned_uint32(sub_block_header+0)); + sb->uncompressed_size = toku_dtoh32(get_unaligned_uint32(sub_block_header+4)); + sb->xsum = toku_dtoh32(get_unaligned_uint32(sub_block_header+8)); +} + +static void verify_block(unsigned char *cp, uint64_t file_offset, uint64_t size) { + // verify the header checksum + const size_t node_header = 8 + sizeof (uint32_t) + sizeof (uint32_t) + sizeof (uint32_t); + + printf("%.8s layout_version=%u %u build=%d\n", cp, get_unaligned_uint32(cp+8), get_unaligned_uint32(cp+12), get_unaligned_uint32(cp+16)); + + unsigned char *sub_block_header = &cp[node_header]; + uint32_t n_sub_blocks = toku_dtoh32(get_unaligned_uint32(&sub_block_header[0])); + uint32_t header_length = node_header + n_sub_blocks * sizeof (struct dump_sub_block); + header_length += sizeof (uint32_t); // CRC + if (header_length > size) { + printf("header length too big: %u\n", header_length); + return; + } + uint32_t header_xsum = toku_x1764_memory(cp, header_length); + uint32_t expected_xsum = toku_dtoh32(get_unaligned_uint32(&cp[header_length])); + if (header_xsum != expected_xsum) { + printf("header checksum failed: %u %u\n", header_xsum, expected_xsum); + return; + } + + // deserialize the sub block header + struct dump_sub_block sub_block[n_sub_blocks]; + sub_block_header += sizeof (uint32_t); + for (uint32_t i = 0 ; i < n_sub_blocks; i++) { + sub_block_deserialize(&sub_block[i], sub_block_header); + sub_block_header += sizeof (struct dump_sub_block); + } + + // verify the sub block header + uint32_t offset = header_length + 4; + for (uint32_t i = 0 ; i < n_sub_blocks; i++) { + uint32_t xsum = toku_x1764_memory(cp + offset, sub_block[i].compressed_size); + printf("%u: %u %u %u", i, sub_block[i].compressed_size, sub_block[i].uncompressed_size, sub_block[i].xsum); + if (xsum != sub_block[i].xsum) + printf(" fail %u offset %" PRIu64, xsum, file_offset + offset); + printf("\n"); + offset += sub_block[i].compressed_size; + } + if (offset != size) + printf("offset %u expected %" PRIu64 "\n", offset, size); +} + +static void dump_block(int fd, BLOCKNUM blocknum, FT ft) { + DISKOFF offset, size; + ft->blocktable.translate_blocknum_to_offset_size(blocknum, &offset, &size); + printf("%" PRId64 " at %" PRId64 " size %" PRId64 "\n", blocknum.b, offset, size); + + unsigned char *CAST_FROM_VOIDP(vp, toku_malloc(size)); + uint64_t r = pread(fd, vp, size, offset); + if (r == (uint64_t)size) { + verify_block(vp, offset, size); + } + toku_free(vp); +} + +static void dump_file(int fd, uint64_t offset, uint64_t size, FILE *outfp) { + unsigned char *XMALLOC_N(size, vp); + uint64_t r = pread(fd, vp, size, offset); + if (r == size) { + if (outfp == stdout) { + hex_dump(vp, offset, size); + } else { + size_t wrote = fwrite(vp, size, 1, outfp); + assert(wrote == 1); + } + } + toku_free(vp); +} + +static void set_file(int fd, uint64_t offset, unsigned char newc) { + toku_os_pwrite(fd, &newc, sizeof newc, offset); +} + +static int readline(char *line, int maxline) { + int i = 0; + int c; + while ((c = getchar()) != EOF && c != '\n' && i < maxline) { + line[i++] = (char)c; + } + line[i++] = 0; + return c == EOF ? EOF : i; +} + +static int split_fields(char *line, char *fields[], int maxfields) { + int i; + for (i=0; i<maxfields; i++) + fields[i] = NULL; + for (i=0; i<maxfields; i++, line=NULL) { + fields[i] = strtok(line, " "); + if (fields[i] == NULL) { + break; + } + } + return i; +} + +static uint64_t getuint64(const char *f) { + if (strncmp(f, "0x", 2) == 0 || strncmp(f, "0X", 2) == 0) + return strtoull(f, 0, 16); + else if (strncmp(f, "0", 1) == 0) + return strtoull(f, 0, 8); + else + return strtoull(f, 0, 10); +} + +static void interactive_help(void) { + fprintf(stderr, "help\n"); + fprintf(stderr, "header\n"); + cout <<"mr/MessagesReport [NUMBER] \n Reports messages for the level of the tree you want get more details about\n"; + cout <<"rf/readFile ft-file-name \n Switch to a different FT\n"; + fprintf(stderr, "node NUMBER \n"); + fprintf(stderr, "bx OFFSET | block_translation OFFSET\n"); + fprintf(stderr, "dumpdata 0|1\n"); + fprintf(stderr, "fragmentation\n"); + fprintf(stderr, "nodesizes\n"); + fprintf(stderr, "garbage\n"); + fprintf(stderr, "file OFFSET SIZE [outfilename]\n"); + fprintf(stderr, "quit\n"); +} + +static void freeNMC(NMC *msgs[], int height){ + for(int i=0;i<height;i++){ + if(msgs[i]!=NULL){ + delete(msgs[i]->count); + + while(msgs[i]->nextNode!=NULL){ + NMC* ptr=msgs[i]->nextNode; + msgs[i]=msgs[i]->nextNode; + delete ptr; + + } + msgs[i]=NULL; + } + } +} + +static void writeTree(NMC *msgs[],int height,char *name UU()){ + ofstream mytree ("/tmp/tree.txt",fstream::out); + if (mytree.is_open()){ + for(int i=height;i>=0;i--){ + NMC * ptr=msgs[i]; + mytree <<i<<endl; + while(ptr!=NULL){ + mytree << ptr->id<<"\t"; + if(ptr->clean!=0)mytree << "1"<<"\t"; + else mytree << "0"<<"\t"; + for(int j=0;j<15;j++)mytree << ptr->count[j]<<" "; + mytree << ptr->count[i]<<endl; + ptr=ptr->nextNode; + } + mytree <<endl; + } + } + else cout << "Unable to open file"; + mytree.close(); +} + +static void writeJson(NMC *msgs[],int height,const char *name){ + ofstream mytree (name,fstream::out); + if (mytree.is_open()){ + mytree <<"{\n \"FT\":["; + for(int i=height;i>=0;i--){ + NMC * ptr=msgs[i]; + mytree <<"{\n\"Level\": {\"Height\":\""<<i<<"\",\n \"Nodes\":["; + while(ptr!=NULL){ + mytree <<"{\"ID\":\""<< ptr->id<<"\","; + if(ptr->clean!=0){ + mytree <<"\"Messages\":["; + for(int j=0;j<16;j++) + { + mytree <<"{"; + switch (j) { + case FT_INSERT: mytree <<"\"INSERT\":\""<<ptr->count[j]<<"\""; break; + case FT_INSERT_NO_OVERWRITE: mytree <<"\"INSERT_NOVERWTE\":\""<<ptr->count[j]<<"\""; break; + case FT_DELETE_ANY: mytree <<"\"DELETE\":\""<<ptr->count[j]<<"\""; break; + case FT_ABORT_ANY: mytree <<"\"ABORT\":\""<<ptr->count[j]<<"\""; break; + case FT_COMMIT_ANY: mytree <<"\"COMMITY\":\""<<ptr->count[j]<<"\""; break; + case FT_COMMIT_BROADCAST_ALL: mytree <<"\"COMMIT_BROADCAST_ALL\":\""<<ptr->count[j]<<"\"" ; break; + case FT_COMMIT_BROADCAST_TXN: mytree <<"\"COMMIT_BROADCAST_TXN\":\""<<ptr->count[j]<<"\""; break; + case FT_ABORT_BROADCAST_TXN: mytree <<"\"ABORT_BROADCAST_TXN\":\""<<ptr->count[j]<<"\"";break; + case FT_OPTIMIZE: mytree <<"\"OPTIMIZE\":\""<<ptr->count[j]<<"\""; break; + case FT_OPTIMIZE_FOR_UPGRADE: mytree <<"\"OPTIMIZE_FOR_UPGRADE\":\""<<ptr->count[j]<<"\"";break; + case FT_UPDATE: mytree <<"\"UPDATE\":\""<<ptr->count[j]<<"\""; break; + case FT_UPDATE_BROADCAST_ALL: mytree <<"\"UPDATE_BROADCAST_ALL\":\""<<ptr->count[j]<<"\""; break; + } + mytree <<"}"; + if(j<15)mytree<<","; + } + + mytree <<"]}"; + + } + else { + mytree <<"\"Messages\":\""<< "0"<<"\"}"; + } + if(ptr->nextNode!=NULL)mytree <<",\n"; + else mytree <<"]}\n"; + ptr=ptr->nextNode; + } + mytree <<"\n}\n"; + if(i!=0)mytree <<",\n"; + } + mytree <<"\n]}\n"; + + } + else cout << "Unable to open file"; + mytree.close(); +} + +static void writeTree(NMC *msgs[],int height){ + ofstream mytree ("/tmp/tree1.txt",fstream::out); + if (mytree.is_open()){ + for(int i=height;i>=0;i--){ + NMC * ptr=msgs[i]; + mytree <<i<<endl; + while(ptr!=NULL){ + mytree << ptr->id<<","; + if(ptr->clean!=0)mytree << "1"<<","; + else mytree << "0"<<","; + for(int j=0;j<15;j++)mytree << ptr->count[j]<<","; + mytree << ptr->count[i]<<endl; + ptr=ptr->nextNode; + } + mytree <<".\""; + } + } + else cout << "Unable to open file"; + mytree.close(); +} + +static void FT_to_JSON(int fd, FT ft, CACHEFILE cf, const char * JsonFile){ + toku_ft_free(ft); + open_header(fd, &ft, cf); + int root=getRootNode(ft); + BLOCKNUM off = make_blocknum(root); + int height=getHeight(fd,off, ft); + NMC *msgs[height]; + for(int i=0;i<=height;i++){ + msgs[i]=NULL; + } + open_header(fd, &ft, cf); + root=getRootNode(ft); + off = make_blocknum(root); + countMessagesInFT(fd,off, ft,msgs); + cout <<"to STD output: \n"; + treeToSTDout(msgs,height); + writeTree(msgs,height); + cout<<"FT's json file was generated here:"; + if(JsonFile!=NULL) { + cout <<JsonFile; + writeJson(msgs,height,JsonFile); + } + else { + cout <<"./FT.json"; + writeJson(msgs,height,"./FT.json"); + } + cout<<endl; + freeNMC(msgs,height); + exit(0); +} + +static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) { + toku_ft_free(ft); + open_header(fd, &ft, cf); + int root=getRootNode(ft); + BLOCKNUM off = make_blocknum(root); + int height=getHeight(fd,off, ft); + NMC *msgs[height]; + for(int i=0;i<=height;i++){ + msgs[i]=NULL; + } + while (1) { + printf("ftdump>"); + fflush(stdout); + char line[maxline+1]; + int r = readline(line, maxline); + if (r == EOF) + break; + const int maxfields = 4; + char *fields[maxfields]; + int nfields = split_fields(line, fields, maxfields); + if (nfields == 0) + continue; + if (strcmp(fields[0], "help") == 0) { + interactive_help(); + } else if (strcmp(fields[0], "header") == 0) { + toku_ft_free(ft); + open_header(fd, &ft, cf); + dump_header(ft); + } else if (strcmp(fields[0], "rn") == 0||strcmp(fields[0], "rootNode")==0||strcmp(fields[0], "rootnode") == 0) { + printf("Root node :%d\n",root); + } else if (strcmp(fields[0], "block") == 0 && nfields == 2) { + BLOCKNUM blocknum = make_blocknum(getuint64(fields[1])); + dump_block(fd, blocknum, ft); + }else if ((strcmp(fields[0], "readFile") == 0 ||strcmp(fields[0], "readfile") == 0 ||strcmp(fields[0], "rf") == 0 )&& nfields == 2) { + fname=fields[1]; + fd = open(fname, O_RDWR + O_BINARY); + toku_ft_free(ft); + open_header(fd, &ft, cf); + root=getRootNode(ft); + off = make_blocknum(root); + height=getHeight(fd,off, ft); + if (fd < 0) { + fprintf(stderr, "%s: can not open the FT dump %s errno %d\n", arg0, fname, errno); + continue; + } + } else if (strcmp(fields[0], "node") == 0 && nfields == 2) { + off = make_blocknum(getuint64(fields[1])); + dump_node(fd, off, ft); + }else if ((strcmp(fields[0], "mr") == 0||(strcmp(fields[0], "nc")) == 0 ||strcmp(fields[0], "messagesReport") == 0 )) { + freeNMC(msgs,height); + toku_ft_free(ft); + open_header(fd, &ft, cf); + root=getRootNode(ft); + off = make_blocknum(root); + countMessagesInFT(fd,off, ft,msgs); + int level=-1; + if(nfields == 2)level=getuint64(fields[1]); + if(level>=0){ + levelToSTDout(msgs[level], level); + } + else{ + cout <<"to STD output: \n"; + treeToSTDout(msgs,height); + } + writeTree(msgs,height); + writeTree(msgs,height, NULL); + + }else if (strcmp(fields[0], "dumpdata") == 0 && nfields == 2) { + + do_dump_data = strtol(fields[1], NULL, 10); + } + else if (strcmp(fields[0], "block_translation") == 0 || strcmp(fields[0], "bx") == 0) { + uint64_t offset = 0; + if (nfields == 2) + offset = getuint64(fields[1]); + dump_block_translation(ft, offset); + } else if (strcmp(fields[0], "fragmentation") == 0) { + dump_fragmentation(fd, ft, do_tsv); + } else if (strcmp(fields[0], "nodesizes") == 0) { + dump_nodesizes(fd, ft); + } else if (strcmp(fields[0], "garbage") == 0||strcmp(fields[0], "g") == 0) { + dump_garbage_stats(fd, ft); + } else if (strcmp(fields[0], "file") == 0 && nfields >= 3) { + uint64_t offset = getuint64(fields[1]); + uint64_t size = getuint64(fields[2]); + FILE *outfp = stdout; + if (nfields >= 4) + outfp = fopen(fields[3], "w"); + dump_file(fd, offset, size, outfp); + } else if (strcmp(fields[0], "setfile") == 0 && nfields == 3) { + uint64_t offset = getuint64(fields[1]); + unsigned char newc = getuint64(fields[2]); + set_file(fd, offset, newc); + } else if (strcmp(fields[0], "quit") == 0 || strcmp(fields[0], "q") == 0) { + toku_ft_free(ft); + exit(0); + } + } + freeNMC(msgs,height); +} + +static int usage(void) { + fprintf(stderr, "Usage: %s ", arg0); + fprintf(stderr, "--interactive "); + fprintf(stderr, "--support /path/to/fractal-tree/file \n\t an interactive way to see what messages and/or switch between FTs"); + fprintf(stderr, "--json /path/to/fractal-tree/file [output json file]\n\t if left empty an FT.json will be created automatically"); + fprintf(stderr, "--nodata "); + fprintf(stderr, "--dumpdata 0|1 "); + fprintf(stderr, "--header "); + fprintf(stderr, "--rootnode "); + fprintf(stderr, "--node N "); + fprintf(stderr, "--fragmentation "); + fprintf(stderr, "--garbage "); + fprintf(stderr, "--tsv "); + fprintf(stderr, "--translation-table "); + fprintf(stderr, "--tsv "); + fprintf(stderr, "--summary "); + fprintf(stderr, "filename \n"); + return 1; +} + +int main (int argc, const char *const argv[]) { + arg0 = argv[0]; + argc--; argv++; + while (argc>0) { + if (strcmp(argv[0], "--interactive") == 0 || strcmp(argv[0], "--i") == 0) { + do_interactive = 1; + } + else if ((strcmp(argv[0], "--json") == 0 || strcmp(argv[0], "--s")== 0)&& argc >= 2) { + do_json = 1; + fname=argv[1]; + argc--; argv++; + break; + } else if (strcmp(argv[0], "--nodata") == 0) { + do_dump_data = 0; + } else if (strcmp(argv[0], "--dumpdata") == 0 && argc > 1) { + do_dump_data = atoi(argv[0]); + } else if (strcmp(argv[0], "--header") == 0) { + do_header = 1; + } else if (strcmp(argv[0], "--rootnode") == 0) { + do_rootnode = 1; + } else if (strcmp(argv[0], "--node") == 0 && argc > 1) { + argc--; argv++; + do_node = 1; + do_node_num = make_blocknum(getuint64(argv[0])); + } else if (strcmp(argv[0], "--fragmentation") == 0) { + do_fragmentation = 1; + } else if (strcmp(argv[0], "--garbage") == 0) { + do_garbage = 1; + } else if (strcmp(argv[0], "--tsv") == 0) { + do_tsv = 1; + } else if (strcmp(argv[0], "--translation-table") == 0) { + do_translation_table = 1; + } else if (strcmp(argv[0], "--summary") == 0) { + do_summary = 1; + } else if (strcmp(argv[0], "--help") == 0 || strcmp(argv[0], "-?") == 0 || strcmp(argv[0], "-h") == 0) { + return usage(); + } else { + break; + } + argc--; argv++; + } + if (argc != 1 && do_json==0) + return usage(); + + int r = toku_ft_layer_init(); + assert_zero(r); + if(fname==NULL)fname = argv[0]; + int fd = open(fname, O_RDWR + O_BINARY); + if (fd < 0) { + fprintf(stderr, "%s: can not open %s errno %d\n", arg0, fname, errno); + return 1; + } + // create a cachefile for the header + CACHETABLE ct = NULL; + toku_cachetable_create(&ct, 1<<25, (LSN){0}, 0); + CACHEFILE cf = NULL; + r = toku_cachetable_openfd (&cf, ct, fd, fname); + assert_zero(r); + FT ft = NULL; + open_header(fd, &ft, cf); + if (do_json ) { + const char *arg=argv[1]; + FT_to_JSON(fd, ft, cf,arg); + } + if (do_interactive) { + run_iteractive_loop(fd, ft, cf); + } + else { + if (do_header) { + dump_header(ft); + } + if (do_rootnode) { + dump_node(fd, ft->h->root_blocknum, ft); + } + if (do_node) { + dump_node(fd, do_node_num, ft); + } + if (do_fragmentation) { + dump_fragmentation(fd, ft, do_tsv); + } + if (do_translation_table) { + ft->blocktable.dump_translation_table_pretty(stdout); + } + if (do_summary) { + dump_summary(fd, ft); + } + if (do_garbage) { + dump_garbage_stats(fd, ft); + } + if (!do_header && !do_rootnode && !do_fragmentation && !do_translation_table && !do_garbage && !do_summary) { + printf("Block translation:"); + ft->blocktable.dump_translation_table(stdout); + dump_header(ft); + struct __dump_node_extra info; + info.fd = fd; + info.ft = ft; + ft->blocktable.iterate(block_table::TRANSLATION_CHECKPOINTED, + dump_node_wrapper, &info, true, true); + } + } + toku_cachefile_close(&cf, false, ZERO_LSN); + toku_cachetable_close(&ct); + toku_ft_free(ft); + toku_ft_layer_destroy(); + return 0; +} |