diff options
Diffstat (limited to 'storage/tokudb/PerconaFT/ftcxx')
20 files changed, 3682 insertions, 0 deletions
diff --git a/storage/tokudb/PerconaFT/ftcxx/CMakeLists.txt b/storage/tokudb/PerconaFT/ftcxx/CMakeLists.txt new file mode 100644 index 00000000..46067803 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/CMakeLists.txt @@ -0,0 +1,31 @@ +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions -Wno-deprecated-declarations") + +if (APPLE) + ## osx is weird about weak symbols + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-flat_namespace,-undefined,dynamic_lookup") +endif (APPLE) + +add_library(ftcxx STATIC + buffer + cursor + db_env + malloc_utils + ) +add_dependencies(ftcxx install_tdb_h) + +if (NOT DEFINED MYSQL_PROJECT_NAME_DOCSTRING) + install( + TARGETS ftcxx + DESTINATION ${INSTALL_LIBDIR} + COMPONENT tokukv_libs_static + ) + + file(GLOB ftcxx_headers "*.hpp") + install( + FILES ${ftcxx_headers} + DESTINATION include/ftcxx + COMPONENT tokukv_headers + ) +endif () + +add_subdirectory(tests) diff --git a/storage/tokudb/PerconaFT/ftcxx/buffer.cpp b/storage/tokudb/PerconaFT/ftcxx/buffer.cpp new file mode 100644 index 00000000..f02014cd --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/buffer.cpp @@ -0,0 +1,141 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <algorithm> +#include <cassert> +#include <cstdlib> +#include <memory> + +#include "buffer.hpp" +#include "malloc_utils.hpp" + +namespace mu = malloc_utils; + +namespace ftcxx { + + const size_t Buffer::INITIAL_CAPACITY = 1<<10; + const size_t Buffer::MAXIMUM_CAPACITY = 1<<18; + const double Buffer::FULLNESS_RATIO = 0.9; + + Buffer::Buffer() + : _cur(0), + _end(0), + _capacity(INITIAL_CAPACITY), + _buf(nullptr, &std::free) + { + init(); + } + + Buffer::Buffer(size_t capacity) + : _end(0), + _capacity(capacity), + _buf(nullptr, &std::free) + { + init(); + } + + char *Buffer::alloc(size_t sz) { + grow(sz); + char *p = raw(_end); + _end += sz; + return p; + } + + bool Buffer::full() const { + return _end > MAXIMUM_CAPACITY * FULLNESS_RATIO; + } + + bool Buffer::more() const { + return _cur < _end; + } + + char *Buffer::current() const { + return raw(_cur); + } + + void Buffer::advance(size_t sz) { + _cur += sz; + } + + void Buffer::clear() { + _cur = 0; + _end = 0; + } + + void Buffer::init() { + _buf.reset(static_cast<char *>(mu::checkedMalloc(_capacity))); + } + + /** + * Implements our growth strategy. Currently we double until we get + * up to 4kB so that we can quickly reach the point where jemalloc can + * help us resize in-place, but after that point we grow by a factor + * of 1.5x. + * + * FBVector doubles once it is bigger than 128kB, but I don't think we + * actually want to because that's about when we want to stop growing. + */ + size_t Buffer::next_alloc_size(size_t sz) { + if (sz < mu::jemallocMinInPlaceExpandable) { + return sz * 2; + } +#if 0 + else if (sz > (128<<10)) { + return sz * 2; + } +#endif + else { + return (sz * 3 + 1) / 2; + } + } + + void Buffer::grow(size_t sz) { + size_t new_capacity = _capacity; + while (new_capacity < _end + sz) { + new_capacity = next_alloc_size(new_capacity); + } + assert(new_capacity >= _capacity); // overflow? + if (new_capacity > _capacity) { + // This section isn't exception-safe, but smartRealloc already + // isn't. The only thing we can throw in here is + // std::bad_alloc, in which case we're kind of screwed anyway. + new_capacity = mu::goodMallocSize(new_capacity); + _buf.reset(static_cast<char *>(mu::smartRealloc(_buf.release(), _end, _capacity, new_capacity, _capacity))); + } + } + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/buffer.hpp b/storage/tokudb/PerconaFT/ftcxx/buffer.hpp new file mode 100644 index 00000000..c0772277 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/buffer.hpp @@ -0,0 +1,159 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <algorithm> +#include <cstdlib> +#include <memory> + +namespace ftcxx { + + /** + * Buffer implements a flat memory buffer intended for FIFO usage + * where allocations are piecemeal but consumption is total. That is, + * we repeatedly fill up the buffer with small allocations, and + * periodically consume all entries and clear the buffer. + * + * For now, the implementation uses a doubling array strategy, + * starting at 1kB growing to a maximum advised capacity of 256kB, + * never shrinking the buffer. + * + * However, we hope to find a better strategy. + * + * Facebook's FBVector claims that a reallocation growth factor of 1.5 + * rather than 2 hits their sweet spot, and they claim to have + * additional improvements by integrating with jemalloc (which we use + * as well). + * + * Additionally, it may be advantageous to use some memarena-style + * tricks like allocating a separate overflow buffer to avoid + * memcpying when we're close to our intended maximum capacity, and + * also to avoid wasting extra memory if we overflow our maximum + * capacity once but never do so again. + */ + class Buffer { + public: + + Buffer(); + + explicit Buffer(size_t capacity); + + Buffer(const Buffer &) = delete; + Buffer& operator=(const Buffer &) = delete; + + Buffer(Buffer&& other) + : _cur(0), + _end(0), + _capacity(0), + _buf(nullptr, &std::free) + { + std::swap(_cur, other._cur); + std::swap(_end, other._end); + std::swap(_capacity, other._capacity); + std::swap(_buf, other._buf); + } + + Buffer& operator=(Buffer&& other) { + std::swap(_cur, other._cur); + std::swap(_end, other._end); + std::swap(_capacity, other._capacity); + std::swap(_buf, other._buf); + return *this; + } + + // Producer API: + + /** + * Allocate room for sz more bytes at the end, and return a + * pointer to the allocated space. This causes at most one + * realloc and memcpy of existing data. + */ + char *alloc(size_t sz); + + /** + * Returns true if we're close to our maximum capacity. If so, + * the producer should stop and allow the consumer to clear the + * buffer. + */ + bool full() const; + + // Consumer API: + + /** + * Returns true if there are more unconsumed bytes in the buffer. + */ + bool more() const; + + /** + * Returns a pointer to the next unconsumed byte in the buffer. + */ + char *current() const; + + /** + * Advances the unconsumed position pointer by sz bytes. + */ + void advance(size_t sz); + + /** + * Free all allocated space. + */ + void clear(); + + private: + + size_t _cur; + size_t _end; + size_t _capacity; + std::unique_ptr<char, void (*)(void*)> _buf; + + static const size_t INITIAL_CAPACITY; + static const size_t MAXIMUM_CAPACITY; + static const double FULLNESS_RATIO; + + void init(); + + static size_t next_alloc_size(size_t sz); + + void grow(size_t sz); + + char *raw(size_t i=0) const { + return &(_buf.get()[i]); + } + }; + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/cursor-inl.hpp b/storage/tokudb/PerconaFT/ftcxx/cursor-inl.hpp new file mode 100644 index 00000000..3ec6787a --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/cursor-inl.hpp @@ -0,0 +1,418 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <algorithm> +#include <cstdint> +#include <utility> + +#include <db.h> + +#include "buffer.hpp" +#include "db_txn.hpp" +#include "exceptions.hpp" +#include "slice.hpp" + +namespace ftcxx { + + class DB; + + template<class Comparator> + bool Bounds::check(Comparator &cmp, const IterationStrategy &strategy, const Slice &key) const { + int c; + if (strategy.forward) { + if (_right_infinite) { + return true; + } + c = cmp(key, _right); + } else { + if (_left_infinite) { + return true; + } + c = cmp(_left, key); + } + if (c > 0 || (c == 0 && _end_exclusive)) { + return false; + } + return true; + } + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler>::CallbackCursor(const DBEnv &env, const DBTxn &txn, + Comparator &&cmp, Handler &&handler) + : _dbc(env, txn), + _iteration_strategy(IterationStrategy(true, true)), + _bounds(DB(env.env()->get_db_for_directory(env.env())), Bounds::Infinite(), Bounds::Infinite(), false), + _cmp(std::forward<Comparator>(cmp)), + _handler(std::forward<Handler>(handler)), + _finished(false) + { + init(); + } + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler>::CallbackCursor(const DB &db, const DBTxn &txn, int flags, + IterationStrategy iteration_strategy, + Bounds bounds, + Comparator &&cmp, Handler &&handler) + : _dbc(db, txn, flags), + _iteration_strategy(iteration_strategy), + _bounds(std::move(bounds)), + _cmp(std::forward<Comparator>(cmp)), + _handler(std::forward<Handler>(handler)), + _finished(false) + { + init(); + } + + template<class Comparator, class Handler> + void CallbackCursor<Comparator, Handler>::init() { + if (!_dbc.set_range(_iteration_strategy, _bounds, getf_callback, this)) { + _finished = true; + } + } + + template<class Comparator, class Handler> + int CallbackCursor<Comparator, Handler>::getf(const DBT *key, const DBT *val) { + if (!_bounds.check(_cmp, _iteration_strategy, Slice(*key))) { + _finished = true; + return -1; + } + + if (!_handler(key, val)) { + return 0; + } + + return TOKUDB_CURSOR_CONTINUE; + } + + template<class Comparator, class Handler> + bool CallbackCursor<Comparator, Handler>::consume_batch() { + if (!_dbc.advance(_iteration_strategy, getf_callback, this)) { + _finished = true; + } + return !_finished; + } + + template<class Comparator, class Handler> + void CallbackCursor<Comparator, Handler>::seek(const Slice &key) { + if (_iteration_strategy.forward) { + _bounds.set_left(key); + } else { + _bounds.set_right(key); + } + if (!_dbc.set_range(_iteration_strategy, _bounds, getf_callback, this)) { + _finished = true; + } + } + + template<class Predicate> + inline void BufferAppender<Predicate>::marshall(char *dest, const DBT *key, const DBT *val) { + uint32_t *keylen = reinterpret_cast<uint32_t *>(&dest[0]); + uint32_t *vallen = reinterpret_cast<uint32_t *>(&dest[sizeof *keylen]); + *keylen = key->size; + *vallen = val->size; + + char *p = &dest[(sizeof *keylen) + (sizeof *vallen)]; + + const char *kp = static_cast<char *>(key->data); + std::copy(kp, kp + key->size, p); + + p += key->size; + + const char *vp = static_cast<char *>(val->data); + std::copy(vp, vp + val->size, p); + } + + template<class Predicate> + inline void BufferAppender<Predicate>::unmarshall(char *src, DBT *key, DBT *val) { + const uint32_t *keylen = reinterpret_cast<uint32_t *>(&src[0]); + const uint32_t *vallen = reinterpret_cast<uint32_t *>(&src[sizeof *keylen]); + key->size = *keylen; + val->size = *vallen; + char *p = &src[(sizeof *keylen) + (sizeof *vallen)]; + key->data = p; + val->data = p + key->size; + } + + template<class Predicate> + inline void BufferAppender<Predicate>::unmarshall(char *src, Slice &key, Slice &val) { + const uint32_t *keylen = reinterpret_cast<uint32_t *>(&src[0]); + const uint32_t *vallen = reinterpret_cast<uint32_t *>(&src[sizeof *keylen]); + char *p = &src[(sizeof *keylen) + (sizeof *vallen)]; + key = Slice(p, *keylen); + val = Slice(p + *keylen, *vallen); + } + + template<class Predicate> + inline bool BufferAppender<Predicate>::operator()(const DBT *key, const DBT *val) { + if (_filter(Slice(*key), Slice(*val))) { + size_t needed = marshalled_size(key->size, val->size); + char *dest = _buf.alloc(needed); + marshall(dest, key, val); + } + return !_buf.full(); + } + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate>::BufferedCursor(const DBEnv &env, const DBTxn &txn, + Comparator &&cmp, Predicate &&filter) + : _buf(), + _cur(env, txn, std::forward<Comparator>(cmp), Appender(_buf, std::forward<Predicate>(filter))) + {} + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate>::BufferedCursor(const DB &db, const DBTxn &txn, int flags, + IterationStrategy iteration_strategy, + Bounds bounds, + Comparator &&cmp, Predicate &&filter) + : _buf(), + _cur(db, txn, flags, + iteration_strategy, + std::move(bounds), + std::forward<Comparator>(cmp), Appender(_buf, std::forward<Predicate>(filter))) + {} + + template<class Comparator, class Predicate> + bool BufferedCursor<Comparator, Predicate>::next(DBT *key, DBT *val) { + if (!_buf.more() && !_cur.finished()) { + _buf.clear(); + _cur.consume_batch(); + } + + if (!_buf.more()) { + return false; + } + + char *src = _buf.current(); + Appender::unmarshall(src, key, val); + _buf.advance(Appender::marshalled_size(key->size, val->size)); + return true; + } + + template<class Comparator, class Predicate> + bool BufferedCursor<Comparator, Predicate>::next(Slice &key, Slice &val) { + if (!_buf.more() && !_cur.finished()) { + _buf.clear(); + _cur.consume_batch(); + } + + if (!_buf.more()) { + return false; + } + + char *src = _buf.current(); + Appender::unmarshall(src, key, val); + _buf.advance(Appender::marshalled_size(key.size(), val.size())); + return true; + } + + template<class Comparator, class Predicate> + void BufferedCursor<Comparator, Predicate>::seek(const Slice &key) { + _buf.clear(); + _cur.seek(key); + } + + template<class Comparator> + SimpleCursor<Comparator>::SimpleCursor(const DBEnv &env, const DBTxn &txn, Comparator &&cmp, + Slice &key, Slice &val) + : _copier(key, val), + _cur(env, txn, std::forward<Comparator>(cmp), _copier) + {} + + template<class Comparator> + SimpleCursor<Comparator>::SimpleCursor(const DB &db, const DBTxn &txn, int flags, + IterationStrategy iteration_strategy, + Bounds bounds, Comparator &&cmp, + Slice &key, Slice &val) + : _copier(key, val), + _cur(db, txn, flags, + iteration_strategy, + std::move(bounds), + std::forward<Comparator>(cmp), _copier) + {} + + template<class Comparator> + bool SimpleCursor<Comparator>::next() { + return _cur.consume_batch(); + } + + template<class Comparator> + void SimpleCursor<Comparator>::seek(const Slice &key) { + _cur.seek(key); + } + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> DB::cursor(const DBTxn &txn, DBT *left, DBT *right, + Comparator &&cmp, Handler &&handler, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return CallbackCursor<Comparator, Handler>(*this, txn, flags, strategy, + Bounds(*this, Slice(*left), Slice(*right), end_exclusive), + std::forward<Comparator>(cmp), std::forward<Handler>(handler)); + } + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> DB::cursor(const DBTxn &txn, const Slice &start_key, + Comparator &&cmp, Handler &&handler, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + Bounds bounds = forward + ? Bounds(*this, start_key, Bounds::Infinite(), end_exclusive) + : Bounds(*this, Bounds::Infinite(), start_key, end_exclusive); + return CallbackCursor<Comparator, Handler>(*this, txn, flags, strategy, std::move(bounds), + std::forward<Comparator>(cmp), std::forward<Handler>(handler)); + } + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> DB::cursor(const DBTxn &txn, const Slice &left, const Slice &right, + Comparator &&cmp, Handler &&handler, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return CallbackCursor<Comparator, Handler>(*this, txn, flags, strategy, + Bounds(*this, left, right, end_exclusive), + std::forward<Comparator>(cmp), std::forward<Handler>(handler)); + } + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> DB::cursor(const DBTxn &txn, Comparator &&cmp, Handler &&handler, + int flags, bool forward, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return CallbackCursor<Comparator, Handler>(*this, txn, flags, strategy, + Bounds(*this, Bounds::Infinite(), Bounds::Infinite(), false), + std::forward<Comparator>(cmp), std::forward<Handler>(handler)); + } + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> DB::buffered_cursor(const DBTxn &txn, DBT *left, DBT *right, + Comparator &&cmp, Predicate &&filter, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return BufferedCursor<Comparator, Predicate>(*this, txn, flags, strategy, + Bounds(*this, Slice(*left), Slice(*right), end_exclusive), + std::forward<Comparator>(cmp), std::forward<Predicate>(filter)); + } + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> DB::buffered_cursor(const DBTxn &txn, const Slice &start_key, + Comparator &&cmp, Predicate &&filter, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + Bounds bounds = forward + ? Bounds(*this, start_key, Bounds::Infinite(), end_exclusive) + : Bounds(*this, Bounds::Infinite(), start_key, end_exclusive); + return BufferedCursor<Comparator, Predicate>(*this, txn, flags, strategy, std::move(bounds), + std::forward<Comparator>(cmp), std::forward<Predicate>(filter)); + } + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> DB::buffered_cursor(const DBTxn &txn, const Slice &left, const Slice &right, + Comparator &&cmp, Predicate &&filter, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return BufferedCursor<Comparator, Predicate>(*this, txn, flags, strategy, + Bounds(*this, left, right, end_exclusive), + std::forward<Comparator>(cmp), std::forward<Predicate>(filter)); + } + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> DB::buffered_cursor(const DBTxn &txn, Comparator &&cmp, Predicate &&filter, + int flags, bool forward, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return BufferedCursor<Comparator, Predicate>(*this, txn, flags, strategy, + Bounds(*this, Bounds::Infinite(), Bounds::Infinite(), false), + std::forward<Comparator>(cmp), std::forward<Predicate>(filter)); + } + + template<class Comparator> + SimpleCursor<Comparator> DB::simple_cursor(const DBTxn &txn, DBT *left, DBT *right, + Comparator &&cmp, Slice &key, Slice &val, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return SimpleCursor<Comparator>(*this, txn, flags, strategy, + Bounds(*this, Slice(*left), Slice(*right), end_exclusive), + std::forward<Comparator>(cmp), key, val); + } + + template<class Comparator> + SimpleCursor<Comparator> DB::simple_cursor(const DBTxn &txn, const Slice &start_key, + Comparator &&cmp, Slice &key, Slice &val, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + Bounds bounds = forward + ? Bounds(*this, start_key, Bounds::Infinite(), end_exclusive) + : Bounds(*this, Bounds::Infinite(), start_key, end_exclusive); + return SimpleCursor<Comparator>(*this, txn, flags, strategy, std::move(bounds), + std::forward<Comparator>(cmp), key, val); + } + + template<class Comparator> + SimpleCursor<Comparator> DB::simple_cursor(const DBTxn &txn, const Slice &left, const Slice &right, + Comparator &&cmp, Slice &key, Slice &val, int flags, + bool forward, bool end_exclusive, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return SimpleCursor<Comparator>(*this, txn, flags, strategy, + Bounds(*this, left, right, end_exclusive), + std::forward<Comparator>(cmp), key, val); + } + + template<class Comparator> + SimpleCursor<Comparator> DB::simple_cursor(const DBTxn &txn, Comparator &&cmp, Slice &key, Slice &val, + int flags, bool forward, bool prelock) const { + IterationStrategy strategy(forward, prelock); + return SimpleCursor<Comparator>(*this, txn, flags, strategy, + Bounds(*this, Bounds::Infinite(), Bounds::Infinite(), false), + std::forward<Comparator>(cmp), key, val); + } + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> DBEnv::cursor(const DBTxn &txn, Comparator &&cmp, Handler &&handler) const { + return CallbackCursor<Comparator, Handler>(*this, txn, std::forward<Comparator>(cmp), std::forward<Handler>(handler)); + } + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> DBEnv::buffered_cursor(const DBTxn &txn, Comparator &&cmp, Predicate &&filter) const { + return BufferedCursor<Comparator, Predicate>(*this, txn, std::forward<Comparator>(cmp), std::forward<Predicate>(filter)); + } + + template<class Comparator> + SimpleCursor<Comparator> DBEnv::simple_cursor(const DBTxn &txn, Comparator &&cmp, Slice &key, Slice &val) const { + return SimpleCursor<Comparator>(*this, txn, std::forward<Comparator>(cmp), key, val); + } + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/cursor.cpp b/storage/tokudb/PerconaFT/ftcxx/cursor.cpp new file mode 100644 index 00000000..e8427bec --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/cursor.cpp @@ -0,0 +1,136 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <db.h> + +#include "cursor.hpp" +#include "db.hpp" +#include "db_env.hpp" +#include "db_txn.hpp" +#include "exceptions.hpp" + +namespace ftcxx { + + DBC::DBC(const DB &db, const DBTxn &txn, int flags) + : _txn(), + _dbc(nullptr) + { + if (db.db() != nullptr) { + DB_TXN *txnp = txn.txn(); + if (txnp == nullptr) { + _txn = DBTxn(DBEnv(db.db()->dbenv), DB_TXN_READ_ONLY | DB_READ_UNCOMMITTED); + txnp = _txn.txn(); + } + + ::DBC *c; + int r = db.db()->cursor(db.db(), txnp, &c, flags); + handle_ft_retval(r); + _dbc = c; + } + } + + DBC::DBC(const DBEnv &env, const DBTxn &txn) + : _txn(), + _dbc(nullptr) + { + if (env.env() != nullptr) { + DB_TXN *txnp = txn.txn(); + if (txnp == nullptr) { + _txn = DBTxn(env, DB_TXN_READ_ONLY | DB_READ_UNCOMMITTED); + txnp = _txn.txn(); + } + + ::DBC *c; + int r = env.env()->get_cursor_for_directory(env.env(), txnp, &c); + handle_ft_retval(r); + _dbc = c; + } + } + + DBC::~DBC() { + if (_dbc != nullptr) { + close(); + } + } + + void DBC::close() { + int r = _dbc->c_close(_dbc); + handle_ft_retval(r); + _dbc = nullptr; + } + + bool DBC::set_range(const IterationStrategy &strategy, const Bounds &bounds, YDB_CALLBACK_FUNCTION callback, void *extra) const { + int r = dbc()->c_set_bounds(dbc(), bounds.left_dbt(), bounds.right_dbt(), strategy.prelock, 0); + handle_ft_retval(r); + + if (strategy.forward) { + if (bounds.left_infinite()) { + r = dbc()->c_getf_first(dbc(), strategy.getf_flags(), callback, extra); + } else { + r = dbc()->c_getf_set_range(dbc(), strategy.getf_flags(), const_cast<DBT *>(bounds.left_dbt()), callback, extra); + } + } else { + if (bounds.right_infinite()) { + r = dbc()->c_getf_last(dbc(), strategy.getf_flags(), callback, extra); + } else { + r = dbc()->c_getf_set_range_reverse(dbc(), strategy.getf_flags(), const_cast<DBT *>(bounds.right_dbt()), callback, extra); + } + } + if (r == DB_NOTFOUND) { + return false; + } else if (r != 0 && r != -1) { + handle_ft_retval(r); + } + return true; + } + + bool DBC::advance(const IterationStrategy &strategy, YDB_CALLBACK_FUNCTION callback, void *extra) const { + int r; + if (strategy.forward) { + r = dbc()->c_getf_next(dbc(), strategy.getf_flags(), callback, extra); + } else { + r = dbc()->c_getf_prev(dbc(), strategy.getf_flags(), callback, extra); + } + if (r == DB_NOTFOUND) { + return false; + } else if (r != 0 && r != -1) { + handle_ft_retval(r); + } + return true; + } + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/cursor.hpp b/storage/tokudb/PerconaFT/ftcxx/cursor.hpp new file mode 100644 index 00000000..bde5dbf2 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/cursor.hpp @@ -0,0 +1,417 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <utility> + +#include <db.h> + +#include "buffer.hpp" +#include "db.hpp" +#include "db_env.hpp" +#include "db_txn.hpp" +#include "slice.hpp" + +namespace ftcxx { + + class DB; + + struct IterationStrategy { + bool forward; + bool prelock; + + IterationStrategy(bool forward_, bool prelock_) + : forward(forward_), + prelock(prelock_) + {} + + int getf_flags() const { + if (prelock) { + return DB_PRELOCKED | DB_PRELOCKED_WRITE; + } else { + return DBC_DISABLE_PREFETCHING; + } + } + }; + + class Bounds { + const ::DB *_db; + Slice _left; + Slice _right; + DBT _left_dbt; + DBT _right_dbt; + bool _left_infinite; + bool _right_infinite; + bool _end_exclusive; + + public: + Bounds(const DB &db, const Slice &left, const Slice &right, bool end_exclusive) + : _db(db.db()), + _left(left.owned()), + _right(right.owned()), + _left_dbt(_left.dbt()), + _right_dbt(_right.dbt()), + _left_infinite(false), + _right_infinite(false), + _end_exclusive(end_exclusive) + {} + + struct Infinite {}; + + Bounds(const DB &db, Infinite, const Slice &right, bool end_exclusive) + : _db(db.db()), + _left(), + _right(right.owned()), + _left_dbt(_left.dbt()), + _right_dbt(_right.dbt()), + _left_infinite(true), + _right_infinite(false), + _end_exclusive(end_exclusive) + {} + + Bounds(const DB &db, const Slice &left, Infinite, bool end_exclusive) + : _db(db.db()), + _left(left.owned()), + _right(), + _left_dbt(_left.dbt()), + _right_dbt(_right.dbt()), + _left_infinite(false), + _right_infinite(true), + _end_exclusive(end_exclusive) + {} + + Bounds(const DB &db, Infinite, Infinite, bool end_exclusive) + : _db(db.db()), + _left(), + _right(), + _left_dbt(_left.dbt()), + _right_dbt(_right.dbt()), + _left_infinite(true), + _right_infinite(true), + _end_exclusive(end_exclusive) + {} + + Bounds(const Bounds &other) = delete; + Bounds& operator=(const Bounds &) = delete; + + Bounds(Bounds &&o) + : _db(nullptr), + _left(), + _right(), + _left_infinite(o._left_infinite), + _right_infinite(o._right_infinite), + _end_exclusive(o._end_exclusive) + { + std::swap(_db, o._db); + std::swap(_left, o._left); + std::swap(_right, o._right); + _left_dbt = _left.dbt(); + _right_dbt = _right.dbt(); + } + + Bounds& operator=(Bounds&& other) { + std::swap(_db, other._db); + std::swap(_left, other._left); + std::swap(_right, other._right); + _left_dbt = _left.dbt(); + _right_dbt = _right.dbt(); + _left_infinite = other._left_infinite; + _right_infinite = other._right_infinite; + _end_exclusive = other._end_exclusive; + return *this; + } + + const DBT *left_dbt() const { + if (_left_infinite) { + return _db->dbt_neg_infty(); + } else { + return &_left_dbt; + } + } + + const DBT *right_dbt() const { + if (_right_infinite) { + return _db->dbt_pos_infty(); + } else { + return &_right_dbt; + } + } + + void set_left(const Slice &left) { + _left = left.owned(); + _left_dbt = _left.dbt(); + _left_infinite = false; + } + + void set_right(const Slice &right) { + _right = right.owned(); + _right_dbt = _right.dbt(); + _right_infinite = false; + } + + bool left_infinite() const { return _left_infinite; } + bool right_infinite() const { return _right_infinite; } + + template<class Comparator> + bool check(Comparator &cmp, const IterationStrategy &strategy, const Slice &key) const; + }; + + /** + * DBC is a simple RAII wrapper around a DBC object. + */ + class DBC { + public: + DBC(const DB &db, const DBTxn &txn=DBTxn(), int flags=0); + ~DBC(); + + // Directory cursor. + DBC(const DBEnv &env, const DBTxn &txn=DBTxn()); + + DBC(const DBC &) = delete; + DBC& operator=(const DBC &) = delete; + + DBC(DBC &&o) + : _txn(), + _dbc(nullptr) + { + std::swap(_txn, o._txn); + std::swap(_dbc, o._dbc); + } + + DBC& operator=(DBC &&o) { + std::swap(_txn, o._txn); + std::swap(_dbc, o._dbc); + return *this; + } + + ::DBC *dbc() const { return _dbc; } + + void set_txn(const DBTxn &txn) const { + _dbc->c_set_txn(_dbc, txn.txn()); + } + + void close(); + + bool set_range(const IterationStrategy &strategy, const Bounds &bounds, YDB_CALLBACK_FUNCTION callback, void *extra) const; + + bool advance(const IterationStrategy &strategy, YDB_CALLBACK_FUNCTION callback, void *extra) const; + + protected: + + // the ordering here matters, for destructors + DBTxn _txn; + ::DBC *_dbc; + }; + + /** + * Cursor supports iterating a cursor over a key range, + * with bulk fetch buffering, and optional filtering. + */ + template<class Comparator, class Handler> + class CallbackCursor { + public: + + /** + * Directory cursor. + */ + CallbackCursor(const DBEnv &env, const DBTxn &txn, + Comparator &&cmp, Handler &&handler); + + /** + * Constructs an cursor. Better to use DB::cursor instead to + * avoid template parameters. + */ + CallbackCursor(const DB &db, const DBTxn &txn, int flags, + IterationStrategy iteration_strategy, + Bounds bounds, + Comparator &&cmp, Handler &&handler); + + /** + * Gets the next key/val pair in the iteration. Returns true + * if there is more data, and fills in key and val. If the + * range is exhausted, returns false. + */ + bool consume_batch(); + + void seek(const Slice &key); + + bool finished() const { return _finished; } + + bool ok() const { return !finished(); } + + void set_txn(const DBTxn &txn) const { _dbc.set_txn(txn); } + + private: + + DBC _dbc; + IterationStrategy _iteration_strategy; + Bounds _bounds; + Comparator _cmp; + Handler _handler; + + bool _finished; + + void init(); + + static int getf_callback(const DBT *key, const DBT *val, void *extra) { + CallbackCursor *i = static_cast<CallbackCursor *>(extra); + return i->getf(key, val); + } + + int getf(const DBT *key, const DBT *val); + }; + + template<class Predicate> + class BufferAppender { + Buffer &_buf; + Predicate _filter; + + public: + BufferAppender(Buffer &buf, Predicate &&filter) + : _buf(buf), + _filter(std::forward<Predicate>(filter)) + {} + + bool operator()(const DBT *key, const DBT *val); + + static size_t marshalled_size(size_t keylen, size_t vallen) { + return (sizeof(((DBT *)0)->size)) + (sizeof(((DBT *)0)->size)) + keylen + vallen; + } + + static void marshall(char *dest, const DBT *key, const DBT *val); + + static void unmarshall(char *src, DBT *key, DBT *val); + static void unmarshall(char *src, Slice &key, Slice &val); + }; + + template<class Comparator, class Predicate> + class BufferedCursor { + public: + + /** + * Directory cursor. + */ + BufferedCursor(const DBEnv &env, const DBTxn &txn, + Comparator &&cmp, Predicate &&filter); + + /** + * Constructs an buffered cursor. Better to use + * DB::buffered_cursor instead to avoid template parameters. + */ + BufferedCursor(const DB &db, const DBTxn &txn, int flags, + IterationStrategy iteration_strategy, + Bounds bounds, + Comparator &&cmp, Predicate &&filter); + + /** + * Gets the next key/val pair in the iteration. Returns true + * if there is more data, and fills in key and val. If the + * range is exhausted, returns false. + */ + bool next(DBT *key, DBT *val); + bool next(Slice &key, Slice &val); + + void seek(const Slice &key); + + bool ok() const { + return _cur.ok() || _buf.more(); + } + + void set_txn(const DBTxn &txn) const { _cur.set_txn(txn); } + + private: + + typedef BufferAppender<Predicate> Appender; + + Buffer _buf; + CallbackCursor<Comparator, Appender> _cur; + }; + + template<class Comparator> + class SimpleCursor { + public: + SimpleCursor(const DBEnv &env, const DBTxn &txn, + Comparator &&cmp, Slice &key, Slice &val); + + SimpleCursor(const DB &db, const DBTxn &txn, int flags, + IterationStrategy iteration_strategy, + Bounds bounds, Comparator &&cmp, + Slice &key, Slice &val); + + /** + * Gets the next key/val pair in the iteration. Copies data + * directly into key and val, which will own their buffers. + */ + bool next(); + + void seek(const Slice &key); + + bool ok() const { + return _cur.ok(); + } + + void set_txn(const DBTxn &txn) const { _cur.set_txn(txn); } + + class SliceCopier { + Slice &_key; + Slice &_val; + + public: + SliceCopier(Slice &key, Slice &val) + : _key(key), + _val(val) + {} + + bool operator()(const DBT *key, const DBT *val) { + _key = Slice(*key).owned(); + _val = Slice(*val).owned(); + + // Don't bulk fetch. + return false; + } + }; + + private: + + SliceCopier _copier; + CallbackCursor<Comparator, SliceCopier&> _cur; + }; + +} // namespace ftcxx + +#include "cursor-inl.hpp" diff --git a/storage/tokudb/PerconaFT/ftcxx/db.hpp b/storage/tokudb/PerconaFT/ftcxx/db.hpp new file mode 100644 index 00000000..7f5af9b6 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/db.hpp @@ -0,0 +1,370 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <string> + +#include <db.h> + +#include "db_env.hpp" +#include "db_txn.hpp" +#include "exceptions.hpp" +#include "slice.hpp" +#include "stats.hpp" + +namespace ftcxx { + + template<class Comparator, class Handler> + class CallbackCursor; + template<class Comparator, class Predicate> + class BufferedCursor; + template<class Comparator> + class SimpleCursor; + + class DB { + public: + DB() + : _db(nullptr), + _close_on_destroy(false) + {} + + explicit DB(::DB *d, bool close_on_destroy=false) + : _db(d), + _close_on_destroy(close_on_destroy) + {} + + ~DB() { + if (_db && _close_on_destroy) { + close(); + } + } + + DB(const DB &) = delete; + DB& operator=(const DB &) = delete; + + DB(DB &&o) + : _db(nullptr), + _close_on_destroy(false) + { + std::swap(_db, o._db); + std::swap(_close_on_destroy, o._close_on_destroy); + } + + DB& operator=(DB &&o) { + std::swap(_db, o._db); + std::swap(_close_on_destroy, o._close_on_destroy); + return *this; + } + + ::DB *db() const { return _db; } + + Slice descriptor() const { + return Slice(_db->cmp_descriptor->dbt); + } + + template<typename Callback> + int getf_set(const DBTxn &txn, const Slice &key, int flags, Callback cb) const { + class WrappedCallback { + Callback &_cb; + public: + WrappedCallback(Callback &cb_) + : _cb(cb_) + {} + + static int call(const DBT *key_, const DBT *val_, void *extra) { + WrappedCallback *wc = static_cast<WrappedCallback *>(extra); + return wc->call(key_, val_); + } + + int call(const DBT *key_, const DBT *val_) { + return _cb(Slice(*key_), Slice(*val_)); + } + } wc(cb); + + DBT kdbt = key.dbt(); + return _db->getf_set(_db, txn.txn(), flags, &kdbt, &WrappedCallback::call, &wc); + } + + int put(const DBTxn &txn, DBT *key, DBT *val, int flags=0) const { + return _db->put(_db, txn.txn(), key, val, flags); + } + + int put(const DBTxn &txn, const Slice &key, const Slice &val, int flags=0) const { + DBT kdbt = key.dbt(); + DBT vdbt = val.dbt(); + return put(txn, &kdbt, &vdbt, flags); + } + + int update(const DBTxn &txn, DBT *key, DBT *val, int flags=0) const { + return _db->update(_db, txn.txn(), key, val, flags); + } + + int update(const DBTxn &txn, const Slice &key, const Slice &extra, int flags=0) const { + DBT kdbt = key.dbt(); + DBT edbt = extra.dbt(); + return update(txn, &kdbt, &edbt, flags); + } + + int del(const DBTxn &txn, DBT *key, int flags=0) const { + return _db->del(_db, txn.txn(), key, flags); + } + + int del(const DBTxn &txn, const Slice &key, int flags=0) const { + DBT kdbt = key.dbt(); + return _db->del(_db, txn.txn(), &kdbt, flags); + } + + template<class OptimizeCallback> + int hot_optimize(const Slice &left, const Slice &right, OptimizeCallback callback, uint64_t *loops_run = NULL) const { + DBT ldbt = left.dbt(); + DBT rdbt = right.dbt(); + + class WrappedOptimizeCallback { + OptimizeCallback &_oc; + size_t _loops; + + public: + WrappedOptimizeCallback(OptimizeCallback &oc) + : _oc(oc), + _loops(0) + {} + + static int call(void *extra, float progress) { + WrappedOptimizeCallback *e = static_cast<WrappedOptimizeCallback *>(extra); + return e->_oc(progress, ++e->_loops); + } + } woc(callback); + + uint64_t dummy; + return _db->hot_optimize(_db, &ldbt, &rdbt, + &WrappedOptimizeCallback::call, &woc, + loops_run == NULL ? &dummy : loops_run); + } + + Stats get_stats() const { + Stats stats; + DB_BTREE_STAT64 s = {0, 0, 0, 0, 0, 0, 0}; + int r = _db->stat64(_db, NULL, &s); + handle_ft_retval(r); + stats.data_size = s.bt_dsize; + stats.file_size = s.bt_fsize; + stats.num_keys = s.bt_nkeys; + return stats; + } + + struct NullFilter { + bool operator()(const Slice &, const Slice &) { + return true; + } + }; + + /** + * Constructs a Cursor over this DB, over the range from left to + * right (or right to left if !forward). + */ + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> cursor(const DBTxn &txn, DBT *left, DBT *right, + Comparator &&cmp, Handler &&handler, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> cursor(const DBTxn &txn, const Slice &start_key, + Comparator &&cmp, Handler &&handler, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> cursor(const DBTxn &txn, const Slice &left, const Slice &right, + Comparator &&cmp, Handler &&handler, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> cursor(const DBTxn &txn, Comparator &&cmp, Handler &&handler, + int flags=0, bool forward=true, bool prelock=false) const; + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> buffered_cursor(const DBTxn &txn, DBT *left, DBT *right, + Comparator &&cmp, Predicate &&filter, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> buffered_cursor(const DBTxn &txn, const Slice &start_key, + Comparator &&cmp, Predicate &&filter, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> buffered_cursor(const DBTxn &txn, const Slice &left, const Slice &right, + Comparator &&cmp, Predicate &&filter, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> buffered_cursor(const DBTxn &txn, Comparator &&cmp, Predicate &&filter, + int flags=0, bool forward=true, bool prelock=false) const; + + template<class Comparator> + SimpleCursor<Comparator> simple_cursor(const DBTxn &txn, DBT *left, DBT *right, + Comparator &&cmp, Slice &key, Slice &val, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator> + SimpleCursor<Comparator> simple_cursor(const DBTxn &txn, const Slice &start_key, + Comparator &&cmp, Slice &key, Slice &val, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator> + SimpleCursor<Comparator> simple_cursor(const DBTxn &txn, const Slice &left, const Slice &right, + Comparator &&cmp, Slice &key, Slice &val, int flags=0, + bool forward=true, bool end_exclusive=false, bool prelock=false) const; + + template<class Comparator> + SimpleCursor<Comparator> simple_cursor(const DBTxn &txn, Comparator &&cmp, Slice &key, Slice &val, + int flags=0, bool forward=true, bool prelock=false) const; + + void close() { + int r = _db->close(_db, 0); + handle_ft_retval(r); + _db = nullptr; + } + + private: + ::DB *_db; + bool _close_on_destroy; + }; + + class DBBuilder { + uint32_t _readpagesize; + int _compression_method; + uint32_t _fanout; + uint8_t _memcmp_magic; + uint32_t _pagesize; + Slice _descriptor; + + public: + DBBuilder() + : _readpagesize(0), + _compression_method(-1), + _fanout(0), + _memcmp_magic(0), + _pagesize(0), + _descriptor() + {} + + DB open(const DBEnv &env, const DBTxn &txn, const char *fname, const char *dbname, DBTYPE dbtype, uint32_t flags, int mode) const { + ::DB *db; + int r = db_create(&db, env.env(), 0); + handle_ft_retval(r); + + if (_readpagesize) { + r = db->set_readpagesize(db, _readpagesize); + handle_ft_retval(r); + } + + if (_compression_method >= 0) { + r = db->set_compression_method(db, TOKU_COMPRESSION_METHOD(_compression_method)); + handle_ft_retval(r); + } + + if (_fanout) { + r = db->set_fanout(db, _fanout); + handle_ft_retval(r); + } + + if (_memcmp_magic) { + r = db->set_memcmp_magic(db, _memcmp_magic); + handle_ft_retval(r); + } + + if (_pagesize) { + r = db->set_pagesize(db, _pagesize); + handle_ft_retval(r); + } + + const DBTxn *txnp = &txn; + DBTxn writeTxn; + if (txn.is_read_only()) { + writeTxn = DBTxn(env, DB_SERIALIZABLE); + txnp = &writeTxn; + } + + r = db->open(db, txnp->txn(), fname, dbname, dbtype, flags, mode); + handle_ft_retval(r); + + if (!_descriptor.empty()) { + DBT desc = _descriptor.dbt(); + r = db->change_descriptor(db, txnp->txn(), &desc, DB_UPDATE_CMP_DESCRIPTOR); + handle_ft_retval(r); + } + + if (txn.is_read_only()) { + writeTxn.commit(); + } + + return DB(db, true); + } + + DBBuilder& set_readpagesize(uint32_t readpagesize) { + _readpagesize = readpagesize; + return *this; + } + + DBBuilder& set_compression_method(TOKU_COMPRESSION_METHOD _compressionmethod) { + _compression_method = int(_compressionmethod); + return *this; + } + + DBBuilder& set_fanout(uint32_t fanout) { + _fanout = fanout; + return *this; + } + + DBBuilder& set_memcmp_magic(uint8_t _memcmpmagic) { + _memcmp_magic = _memcmpmagic; + return *this; + } + + DBBuilder& set_pagesize(uint32_t pagesize) { + _pagesize = pagesize; + return *this; + } + + DBBuilder& set_descriptor(const Slice &desc) { + _descriptor = desc.owned(); + return *this; + } + }; + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/db_env-inl.hpp b/storage/tokudb/PerconaFT/ftcxx/db_env-inl.hpp new file mode 100644 index 00000000..43ab86ec --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/db_env-inl.hpp @@ -0,0 +1,75 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <db.h> + +#include "db.hpp" +#include "slice.hpp" + +namespace ftcxx { + + typedef int (*slice_compare_func)(const Slice &desc, const Slice &key, const Slice &val); + + template<slice_compare_func slice_cmp> + int wrapped_comparator(::DB *db, const DBT *a, const DBT *b) { + return slice_cmp(DB(db).descriptor(), Slice(*a), Slice(*b)); + } + + class SetvalFunc { + void (*_setval)(const DBT *, void *); + void *_extra; + public: + SetvalFunc(void (*setval)(const DBT *, void *), void *extra) + : _setval(setval), + _extra(extra) + {} + void operator()(const Slice &new_val) { + DBT vdbt = new_val.dbt(); + _setval(&vdbt, _extra); + } + }; + + typedef int (*slice_update_func)(const Slice &desc, const Slice &key, const Slice &old_val, const Slice &extra, SetvalFunc callback); + + template<slice_update_func slice_update> + int wrapped_updater(::DB *db, const DBT *key, const DBT *old_val, const DBT *extra, void (*setval)(const DBT *, void *), void *setval_extra) { + return slice_update(DB(db).descriptor(), Slice(*key), Slice(*old_val), Slice(*extra), SetvalFunc(setval, setval_extra)); + } + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/db_env.cpp b/storage/tokudb/PerconaFT/ftcxx/db_env.cpp new file mode 100644 index 00000000..bb8df22c --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/db_env.cpp @@ -0,0 +1,70 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <map> +#include <memory> +#include <string> + +#include <db.h> + +#include "db_env.hpp" + +namespace ftcxx { + + void DBEnv::get_status(DBEnv::Status &status, fs_redzone_state &redzone_state, uint64_t &env_panic, std::string &panic_string) const{ + uint64_t num_rows; + int r = _env->get_engine_status_num_rows(_env, &num_rows); + handle_ft_retval(r); + + std::unique_ptr<TOKU_ENGINE_STATUS_ROW_S[]> buf(new TOKU_ENGINE_STATUS_ROW_S[num_rows]); + char panic_string_buf[1<<12]; + panic_string_buf[0] = '\0'; + + r = _env->get_engine_status(_env, buf.get(), num_rows, &num_rows, + &redzone_state, + &env_panic, panic_string_buf, sizeof panic_string_buf, + toku_engine_status_include_type(TOKU_ENGINE_STATUS | TOKU_GLOBAL_STATUS)); + handle_ft_retval(r); + + panic_string = std::string(panic_string_buf); + + for (uint64_t i = 0; i < num_rows; ++i) { + status[buf[i].keyname] = buf[i]; + } + } + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/db_env.hpp b/storage/tokudb/PerconaFT/ftcxx/db_env.hpp new file mode 100644 index 00000000..15b5ce55 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/db_env.hpp @@ -0,0 +1,466 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <errno.h> + +#include <map> +#include <string> + +#include <db.h> + +#include "exceptions.hpp" +#include "slice.hpp" + +namespace ftcxx { + + template<class Comparator, class Handler> + class CallbackCursor; + template<class Comparator, class Predicate> + class BufferedCursor; + template<class Comparator> + class SimpleCursor; + + class DBTxn; + + class DBEnv { + public: + explicit DBEnv(DB_ENV *e, bool close_on_destroy=false) + : _env(e), + _close_on_destroy(close_on_destroy) + {} + + ~DBEnv() { + if (_env && _close_on_destroy) { + close(); + } + } + + DBEnv(const DBEnv &) = delete; + DBEnv& operator=(const DBEnv &) = delete; + + DBEnv(DBEnv &&o) + : _env(nullptr), + _close_on_destroy(false) + { + std::swap(_env, o._env); + std::swap(_close_on_destroy, o._close_on_destroy); + } + + DBEnv& operator=(DBEnv &&o) { + std::swap(_env, o._env); + std::swap(_close_on_destroy, o._close_on_destroy); + return *this; + } + + DB_ENV *env() const { return _env; } + + void close() { + int r = _env->close(_env, 0); + handle_ft_retval(r); + _env = nullptr; + } + + typedef std::map<std::string, TOKU_ENGINE_STATUS_ROW_S> Status; + void get_status(Status &status, fs_redzone_state &redzone_state, uint64_t &env_panic, std::string &panic_string) const; + + void log_flush() { + int r = _env->log_flush(_env, NULL); + handle_ft_retval(r); + } + + int checkpointing_set_period(uint32_t period) { + if (!_env) { + return EINVAL; + } + _env->checkpointing_set_period(_env, period); + return 0; + } + + int cleaner_set_iterations(uint32_t iterations) { + if (!_env) { + return EINVAL; + } + _env->cleaner_set_iterations(_env, iterations); + return 0; + } + + int cleaner_set_period(uint32_t period) { + if (!_env) { + return EINVAL; + } + _env->cleaner_set_period(_env, period); + return 0; + } + + int change_fsync_log_period(uint32_t period) { + if (!_env) { + return EINVAL; + } + _env->change_fsync_log_period(_env, period); + return 0; + } + + uint64_t get_engine_status_num_rows() { + if (!_env) { + handle_ft_retval(EINVAL); // throws + } + uint64_t ret; + int r = _env->get_engine_status_num_rows(_env, &ret); + handle_ft_retval(r); + return ret; + } + + void get_engine_status(TOKU_ENGINE_STATUS_ROW_S *rows, uint64_t max_rows, uint64_t &num_rows, + uint64_t &panic, std::string &panic_string, + toku_engine_status_include_type include_type) { + if (!_env) { + handle_ft_retval(EINVAL); + } + fs_redzone_state dummy; // this is duplicated in the actual engine status output + const size_t panic_string_len = 1024; + char panic_string_buf[panic_string_len]; + panic_string_buf[0] = '\0'; + int r = _env->get_engine_status(_env, rows, max_rows, &num_rows, + &dummy, &panic, panic_string_buf, panic_string_len, + include_type); + handle_ft_retval(r); + panic_string = panic_string_buf; + } + + /** + * Constructs a Cursor over this DBEnv's directory. + */ + template<class Comparator, class Handler> + CallbackCursor<Comparator, Handler> cursor(const DBTxn &txn, Comparator &&cmp, Handler &&handler) const; + + template<class Comparator, class Predicate> + BufferedCursor<Comparator, Predicate> buffered_cursor(const DBTxn &txn, Comparator &&cmp, Predicate &&filter) const; + + template<class Comparator> + SimpleCursor<Comparator> simple_cursor(const DBTxn &txn, Comparator &&cmp, Slice &key, Slice &val) const; + + private: + DB_ENV *_env; + bool _close_on_destroy; + }; + + class DBEnvBuilder { + typedef int (*bt_compare_func)(DB *, const DBT *, const DBT *); + bt_compare_func _bt_compare; + + typedef int (*update_func)(DB *, const DBT *, const DBT *, const DBT *, void (*)(const DBT *, void *), void *); + update_func _update_function; + + generate_row_for_put_func _generate_row_for_put; + generate_row_for_del_func _generate_row_for_del; + + uint32_t _cleaner_period; + uint32_t _cleaner_iterations; + uint32_t _checkpointing_period; + uint32_t _fsync_log_period_msec; + int _fs_redzone; + + uint64_t _lk_max_memory; + uint64_t _lock_wait_time_msec; + + typedef uint64_t (*get_lock_wait_time_cb_func)(uint64_t); + get_lock_wait_time_cb_func _get_lock_wait_time_cb; + lock_timeout_callback _lock_timeout_callback; + lock_wait_callback _lock_wait_needed_callback; + uint64_t (*_loader_memory_size_callback)(void); + + uint32_t _cachesize_gbytes; + uint32_t _cachesize_bytes; + uint32_t _cachetable_bucket_mutexes; + + std::string _product_name; + + std::string _lg_dir; + std::string _tmp_dir; + + bool _direct_io; + bool _compress_buffers; + + public: + DBEnvBuilder() + : _bt_compare(nullptr), + _update_function(nullptr), + _generate_row_for_put(nullptr), + _generate_row_for_del(nullptr), + _cleaner_period(0), + _cleaner_iterations(0), + _checkpointing_period(0), + _fsync_log_period_msec(0), + _fs_redzone(0), + _lk_max_memory(0), + _lock_wait_time_msec(0), + _get_lock_wait_time_cb(nullptr), + _lock_timeout_callback(nullptr), + _lock_wait_needed_callback(nullptr), + _loader_memory_size_callback(nullptr), + _cachesize_gbytes(0), + _cachesize_bytes(0), + _cachetable_bucket_mutexes(0), + _product_name(""), + _lg_dir(""), + _tmp_dir(""), + _direct_io(false), + _compress_buffers(true) + {} + + DBEnv open(const char *env_dir, uint32_t flags, int mode) const { + db_env_set_direct_io(_direct_io); + db_env_set_compress_buffers_before_eviction(_compress_buffers); + if (_cachetable_bucket_mutexes) { + db_env_set_num_bucket_mutexes(_cachetable_bucket_mutexes); + } + + if (!_product_name.empty()) { + db_env_set_toku_product_name(_product_name.c_str()); + } + + DB_ENV *env; + int r = db_env_create(&env, 0); + handle_ft_retval(r); + + if (_bt_compare) { + r = env->set_default_bt_compare(env, _bt_compare); + handle_ft_retval(r); + } + + if (_update_function) { + env->set_update(env, _update_function); + } + + if (_generate_row_for_put) { + r = env->set_generate_row_callback_for_put(env, _generate_row_for_put); + handle_ft_retval(r); + } + + if (_generate_row_for_del) { + r = env->set_generate_row_callback_for_del(env, _generate_row_for_del); + handle_ft_retval(r); + } + + if (_lk_max_memory) { + r = env->set_lk_max_memory(env, _lk_max_memory); + handle_ft_retval(r); + } + + if (_lock_wait_time_msec || _get_lock_wait_time_cb) { + uint64_t wait_time = _lock_wait_time_msec; + if (!wait_time) { + r = env->get_lock_timeout(env, &wait_time); + handle_ft_retval(r); + } + r = env->set_lock_timeout(env, wait_time, _get_lock_wait_time_cb); + handle_ft_retval(r); + } + + if (_lock_timeout_callback) { + r = env->set_lock_timeout_callback(env, _lock_timeout_callback); + handle_ft_retval(r); + } + + if (_lock_wait_needed_callback) { + r = env->set_lock_wait_callback(env, _lock_wait_needed_callback); + handle_ft_retval(r); + } + + if (_loader_memory_size_callback) { + env->set_loader_memory_size(env, _loader_memory_size_callback); + } + + if (_cachesize_gbytes || _cachesize_bytes) { + r = env->set_cachesize(env, _cachesize_gbytes, _cachesize_bytes, 1); + handle_ft_retval(r); + } + + if (_fs_redzone) { + env->set_redzone(env, _fs_redzone); + } + + if (!_lg_dir.empty()) { + r = env->set_lg_dir(env, _lg_dir.c_str()); + handle_ft_retval(r); + } + + if (!_tmp_dir.empty()) { + r = env->set_tmp_dir(env, _tmp_dir.c_str()); + handle_ft_retval(r); + } + + r = env->open(env, env_dir, flags, mode); + handle_ft_retval(r); + + if (_cleaner_period) { + r = env->cleaner_set_period(env, _cleaner_period); + handle_ft_retval(r); + } + + if (_cleaner_iterations) { + r = env->cleaner_set_iterations(env, _cleaner_iterations); + handle_ft_retval(r); + } + + if (_checkpointing_period) { + r = env->checkpointing_set_period(env, _checkpointing_period); + handle_ft_retval(r); + } + + if (_fsync_log_period_msec) { + env->change_fsync_log_period(env, _fsync_log_period_msec); + } + + return DBEnv(env, true); + } + + DBEnvBuilder& set_direct_io(bool direct_io) { + _direct_io = direct_io; + return *this; + } + + DBEnvBuilder& set_compress_buffers_before_eviction(bool compress_buffers) { + _compress_buffers = compress_buffers; + return *this; + } + + DBEnvBuilder& set_default_bt_compare(bt_compare_func bt_compare) { + _bt_compare = bt_compare; + return *this; + } + + DBEnvBuilder& set_update(update_func update_function) { + _update_function = update_function; + return *this; + } + + DBEnvBuilder& set_generate_row_callback_for_put(generate_row_for_put_func generate_row_for_put) { + _generate_row_for_put = generate_row_for_put; + return *this; + } + + DBEnvBuilder& set_generate_row_callback_for_del(generate_row_for_del_func generate_row_for_del) { + _generate_row_for_del = generate_row_for_del; + return *this; + } + + DBEnvBuilder& cleaner_set_period(uint32_t period) { + _cleaner_period = period; + return *this; + } + + DBEnvBuilder& cleaner_set_iterations(uint32_t iterations) { + _cleaner_iterations = iterations; + return *this; + } + + DBEnvBuilder& checkpointing_set_period(uint32_t period) { + _checkpointing_period = period; + return *this; + } + + DBEnvBuilder& change_fsync_log_period(uint32_t period) { + _fsync_log_period_msec = period; + return *this; + } + + DBEnvBuilder& set_fs_redzone(int fs_redzone) { + _fs_redzone = fs_redzone; + return *this; + } + + DBEnvBuilder& set_lk_max_memory(uint64_t sz) { + _lk_max_memory = sz; + return *this; + } + + DBEnvBuilder& set_lock_wait_time_msec(uint64_t lock_wait_time_msec) { + _lock_wait_time_msec = lock_wait_time_msec; + return *this; + } + + DBEnvBuilder& set_lock_wait_time_cb(get_lock_wait_time_cb_func get_lock_wait_time_cb) { + _get_lock_wait_time_cb = get_lock_wait_time_cb; + return *this; + } + + DBEnvBuilder& set_lock_timeout_callback(lock_timeout_callback callback) { + _lock_timeout_callback = callback; + return *this; + } + + DBEnvBuilder& set_lock_wait_callback(lock_wait_callback callback) { + _lock_wait_needed_callback = callback; + return *this; + } + + DBEnvBuilder& set_loader_memory_size(uint64_t (*callback)(void)) { + _loader_memory_size_callback = callback; + return *this; + } + + DBEnvBuilder& set_cachesize(uint32_t gbytes, uint32_t bytes) { + _cachesize_gbytes = gbytes; + _cachesize_bytes = bytes; + return *this; + } + + DBEnvBuilder& set_cachetable_bucket_mutexes(uint32_t mutexes) { + _cachetable_bucket_mutexes = mutexes; + return *this; + } + + DBEnvBuilder& set_product_name(const char *product_name) { + _product_name = std::string(product_name); + return *this; + } + + DBEnvBuilder& set_lg_dir(const char *lg_dir) { + _lg_dir = std::string(lg_dir); + return *this; + } + + DBEnvBuilder& set_tmp_dir(const char *tmp_dir) { + _tmp_dir = std::string(tmp_dir); + return *this; + } + }; + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/db_txn.hpp b/storage/tokudb/PerconaFT/ftcxx/db_txn.hpp new file mode 100644 index 00000000..adcdc8f5 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/db_txn.hpp @@ -0,0 +1,127 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <db.h> + +#include "db_env.hpp" +#include "exceptions.hpp" + +namespace ftcxx { + + class DBTxn { + public: + DBTxn() + : _flags(0), + _txn(nullptr) + {} + + explicit DBTxn(const DBEnv &env, int flags=0) + : _flags(flags), + _txn(nullptr) + { + DB_TXN *t; + int r = env.env()->txn_begin(env.env(), nullptr, &t, _flags); + handle_ft_retval(r); + _txn = t; + } + + DBTxn(const DBEnv &env, const DBTxn &parent, int flags=0) + : _flags(flags), + _txn(nullptr) + { + DB_TXN *t; + int r = env.env()->txn_begin(env.env(), parent.txn(), &t, _flags); + handle_ft_retval(r); + _txn = t; + } + + ~DBTxn() { + if (_txn) { + abort(); + } + } + + DBTxn(const DBTxn &) = delete; + DBTxn& operator=(const DBTxn &) = delete; + + DBTxn(DBTxn &&o) + : _flags(0), + _txn(nullptr) + { + std::swap(_flags, o._flags); + std::swap(_txn, o._txn); + } + + DBTxn& operator=(DBTxn &&o) { + std::swap(_flags, o._flags); + std::swap(_txn, o._txn); + return *this; + } + + DB_TXN *txn() const { return _txn; } + + void commit(int flags=0) { + int r = _txn->commit(_txn, flags); + handle_ft_retval(r); + _txn = nullptr; + } + + void abort() { + int r = _txn->abort(_txn); + handle_ft_retval(r); + _txn = nullptr; + } + + bool is_read_only() const { + return _flags & DB_TXN_READ_ONLY; + } + + uint64_t id() const { + if (!_txn) { + return 0; + } + return _txn->id64(_txn); + } + + private: + int _flags; + DB_TXN *_txn; + }; + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/exceptions.hpp b/storage/tokudb/PerconaFT/ftcxx/exceptions.hpp new file mode 100644 index 00000000..d8080d41 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/exceptions.hpp @@ -0,0 +1,152 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <exception> +#include <string.h> + +#include <db.h> + +namespace ftcxx { + + class ft_exception : public std::exception { + int _code; + + static const char *ft_strerror(int code) { + switch (code) { + case DB_RUNRECOVERY: + return "DB_RUNRECOVERY"; + case DB_KEYEXIST: + return "DB_KEYEXIST"; + case DB_LOCK_DEADLOCK: + return "DB_LOCK_DEADLOCK"; + case DB_LOCK_NOTGRANTED: + return "DB_LOCK_NOTGRANTED"; + case DB_NOTFOUND: + return "DB_NOTFOUND"; + case DB_SECONDARY_BAD: + return "DB_SECONDARY_BAD"; + case DB_DONOTINDEX: + return "DB_DONOTINDEX"; + case DB_BUFFER_SMALL: + return "DB_BUFFER_SMALL"; + case DB_BADFORMAT: + return "DB_BADFORMAT"; + case TOKUDB_OUT_OF_LOCKS: + return "TOKUDB_OUT_OF_LOCKS"; + case TOKUDB_SUCCEEDED_EARLY: + return "TOKUDB_SUCCEEDED_EARLY"; + case TOKUDB_FOUND_BUT_REJECTED: + return "TOKUDB_FOUND_BUT_REJECTED"; + case TOKUDB_USER_CALLBACK_ERROR: + return "TOKUDB_USER_CALLBACK_ERROR"; + case TOKUDB_DICTIONARY_TOO_OLD: + return "TOKUDB_DICTIONARY_TOO_OLD"; + case TOKUDB_DICTIONARY_TOO_NEW: + return "TOKUDB_DICTIONARY_TOO_NEW"; + case TOKUDB_DICTIONARY_NO_HEADER: + return "TOKUDB_DICTIONARY_NO_HEADER"; + case TOKUDB_CANCELED: + return "TOKUDB_CANCELED"; + case TOKUDB_NO_DATA: + return "TOKUDB_NO_DATA"; + case TOKUDB_ACCEPT: + return "TOKUDB_ACCEPT"; + case TOKUDB_MVCC_DICTIONARY_TOO_NEW: + return "TOKUDB_MVCC_DICTIONARY_TOO_NEW"; + case TOKUDB_UPGRADE_FAILURE: + return "TOKUDB_UPGRADE_FAILURE"; + case TOKUDB_TRY_AGAIN: + return "TOKUDB_TRY_AGAIN"; + case TOKUDB_NEEDS_REPAIR: + return "TOKUDB_NEEDS_REPAIR"; + case TOKUDB_CURSOR_CONTINUE: + return "TOKUDB_CURSOR_CONTINUE"; + case TOKUDB_BAD_CHECKSUM: + return "TOKUDB_BAD_CHECKSUM"; + case TOKUDB_HUGE_PAGES_ENABLED: + return "TOKUDB_HUGE_PAGES_ENABLED"; + case TOKUDB_OUT_OF_RANGE: + return "TOKUDB_OUT_OF_RANGE"; + case TOKUDB_INTERRUPTED: + return "TOKUDB_INTERRUPTED"; + default: + return "unknown ft error"; + } + } + + public: + ft_exception(int c) : _code(c) {} + + int code() const noexcept { + return _code; + } + + virtual const char *what() const noexcept { + return ft_strerror(_code); + } + }; + + class system_exception : public std::exception { + int _code; + + public: + system_exception(int c) : _code(c) {} + + int code() const noexcept { + return _code; + } + + virtual const char *what() const noexcept { + return strerror(_code); + } + }; + + inline void handle_ft_retval(int r) { + if (r == 0) { + return; + } + if (r < 0) { + throw ft_exception(r); + } + if (r > 0) { + throw system_exception(r); + } + } + +} // namespace ftcxx diff --git a/storage/tokudb/PerconaFT/ftcxx/malloc_utils.cpp b/storage/tokudb/PerconaFT/ftcxx/malloc_utils.cpp new file mode 100644 index 00000000..6c0fb341 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/malloc_utils.cpp @@ -0,0 +1,97 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <cstdint> + +#include "malloc_utils.hpp" + +#if !defined(HAVE_BITS_FUNCTEXCEPT_H) || !HAVE_BITS_FUNCTEXCEPT_H + +namespace std { + + void __throw_bad_alloc() { + throw bad_alloc(); + } + +} // namespace std + +#endif + +namespace malloc_utils { + + // How do we determine that we're using jemalloc? + // In the hackiest way possible. We allocate memory using malloc() and see if + // the per-thread counter of allocated memory increases. This makes me feel + // dirty inside. Also note that this requires jemalloc to have been compiled + // with --enable-stats. + bool usingJEMallocSlow() { + // Some platforms (*cough* OSX *cough*) require weak symbol checks to be + // in the form if (mallctl != nullptr). Not if (mallctl) or if (!mallctl) + // (!!). http://goo.gl/xpmctm + if (mallocx == nullptr || rallocx == nullptr || xallocx == nullptr + || sallocx == nullptr || dallocx == nullptr || nallocx == nullptr + || mallctl == nullptr) { + return false; + } + + // "volatile" because gcc optimizes out the reads from *counter, because + // it "knows" malloc doesn't modify global state... + volatile uint64_t* counter; + size_t counterLen = sizeof(uint64_t*); + + if (mallctl("thread.allocatedp", static_cast<void*>(&counter), &counterLen, + nullptr, 0) != 0) { + return false; + } + + if (counterLen != sizeof(uint64_t*)) { + return false; + } + + uint64_t origAllocated = *counter; + + void* ptr = malloc(1); + if (!ptr) { + // wtf, failing to allocate 1 byte + return false; + } + free(ptr); + + return (origAllocated != *counter); + } + +} // namespace malloc_utils diff --git a/storage/tokudb/PerconaFT/ftcxx/malloc_utils.hpp b/storage/tokudb/PerconaFT/ftcxx/malloc_utils.hpp new file mode 100644 index 00000000..4aae801b --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/malloc_utils.hpp @@ -0,0 +1,226 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +/** + * These functions are extracted from Facebook's folly library, which + * integrates well with jemalloc. See + * https://github.com/facebook/folly/blob/master/folly/Malloc.h + */ + +#include <algorithm> +#include <cassert> +#include <cstdlib> + +#if defined(HAVE_BITS_FUNCTEXCEPT_H) && HAVE_BITS_FUNCTEXCEPT_H + +# include <bits/functexcept.h> + +#else + +# include <stdexcept> + +namespace std { + + void __throw_bad_alloc(); + +} + +#endif + +/** + * Declare *allocx() and mallctl() as weak symbols. These will be provided by + * jemalloc if we are using jemalloc, or will be NULL if we are using another + * malloc implementation. + */ +extern "C" void* mallocx(size_t, int) + __attribute__((__weak__)); +extern "C" void* rallocx(void*, size_t, int) + __attribute__((__weak__)); +extern "C" size_t xallocx(void*, size_t, size_t, int) + __attribute__((__weak__)); +extern "C" size_t sallocx(const void*, int) + __attribute__((__weak__)); +extern "C" void dallocx(void*, int) + __attribute__((__weak__)); +extern "C" size_t nallocx(size_t, int) + __attribute__((__weak__)); +extern "C" int mallctl(const char*, void*, size_t*, void*, size_t) + __attribute__((__weak__)); + +namespace malloc_utils { + + bool usingJEMallocSlow(); + + /** + * Determine if we are using jemalloc or not. + */ + inline bool usingJEMalloc() { + // Checking for rallocx != NULL is not sufficient; we may be in a + // dlopen()ed module that depends on libjemalloc, so rallocx is + // resolved, but the main program might be using a different + // memory allocator. Look at the implementation of + // usingJEMallocSlow() for the (hacky) details. + static const bool result = usingJEMallocSlow(); + return result; + } + + /** + * For jemalloc's size classes, see + * http://www.canonware.com/download/jemalloc/jemalloc-latest/doc/jemalloc.html + */ + inline size_t goodMallocSize(size_t minSize) noexcept { + if (!usingJEMalloc()) { + // Not using jemalloc - no smarts + return minSize; + } + size_t goodSize; + if (minSize <= 64) { + // Choose smallest allocation to be 64 bytes - no tripping + // over cache line boundaries, and small string optimization + // takes care of short strings anyway. + goodSize = 64; + } else if (minSize <= 512) { + // Round up to the next multiple of 64; we don't want to trip + // over cache line boundaries. + goodSize = (minSize + 63) & ~size_t(63); + } else if (minSize <= 3584) { + // Round up to the next multiple of 256. For some size + // classes jemalloc will additionally round up to the nearest + // multiple of 512, hence the nallocx() call. + goodSize = nallocx((minSize + 255) & ~size_t(255), 0); + } else if (minSize <= 4072 * 1024) { + // Round up to the next multiple of 4KB + goodSize = (minSize + 4095) & ~size_t(4095); + } else { + // Holy Moly + // Round up to the next multiple of 4MB + goodSize = (minSize + 4194303) & ~size_t(4194303); + } + assert(nallocx(goodSize, 0) == goodSize); + return goodSize; + } + + static const size_t jemallocMinInPlaceExpandable = 4096; + + /** + * Trivial wrappers around malloc, calloc, realloc that check for + * allocation failure and throw std::bad_alloc in that case. + */ + inline void* checkedMalloc(size_t size) { + void* p = malloc(size); + if (!p) std::__throw_bad_alloc(); + return p; + } + + inline void* checkedCalloc(size_t n, size_t size) { + void* p = calloc(n, size); + if (!p) std::__throw_bad_alloc(); + return p; + } + + inline void* checkedRealloc(void* ptr, size_t size) { + void* p = realloc(ptr, size); + if (!p) std::__throw_bad_alloc(); + return p; + } + + /** + * This function tries to reallocate a buffer of which only the first + * currentSize bytes are used. The problem with using realloc is that + * if currentSize is relatively small _and_ if realloc decides it + * needs to move the memory chunk to a new buffer, then realloc ends + * up copying data that is not used. It's impossible to hook into + * GNU's malloc to figure whether expansion will occur in-place or as + * a malloc-copy-free troika. (If an expand_in_place primitive would + * be available, smartRealloc would use it.) As things stand, this + * routine just tries to call realloc() (thus benefitting of potential + * copy-free coalescing) unless there's too much slack memory. + */ + inline void* smartRealloc(void* p, + const size_t currentSize, + const size_t currentCapacity, + const size_t newCapacity, + size_t &realNewCapacity) { + assert(p); + assert(currentSize <= currentCapacity && + currentCapacity < newCapacity); + + if (usingJEMalloc()) { + // using jemalloc's API. Don't forget that jemalloc can never + // grow in place blocks smaller than 4096 bytes. + // + // NB: newCapacity may not be precisely equal to a jemalloc + // size class, i.e. newCapacity is not guaranteed to be the + // result of a goodMallocSize() call, therefore xallocx() may + // return more than newCapacity bytes of space. Use >= rather + // than == to check whether xallocx() successfully expanded in + // place. + size_t realNewCapacity_; + if (currentCapacity >= jemallocMinInPlaceExpandable && + (realNewCapacity_ = xallocx(p, newCapacity, 0, 0)) >= newCapacity) { + // Managed to expand in place + realNewCapacity = realNewCapacity_; + return p; + } + // Cannot expand; must move + char * const result = static_cast<char *>(checkedMalloc(newCapacity)); + char *cp = static_cast<char *>(p); + std::copy(cp, cp + currentSize, result); + free(p); + realNewCapacity = newCapacity; + return result; + } + + // No jemalloc no honey + auto const slack = currentCapacity - currentSize; + if (slack * 2 > currentSize) { + // Too much slack, malloc-copy-free cycle: + char * const result = static_cast<char *>(checkedMalloc(newCapacity)); + char *cp = static_cast<char *>(p); + std::copy(cp, cp + currentSize, result); + free(p); + realNewCapacity = newCapacity; + return result; + } + // If there's not too much slack, we realloc in hope of coalescing + realNewCapacity = newCapacity; + return checkedRealloc(p, newCapacity); + } + +} // namespace malloc_utils diff --git a/storage/tokudb/PerconaFT/ftcxx/slice.hpp b/storage/tokudb/PerconaFT/ftcxx/slice.hpp new file mode 100644 index 00000000..138d8a4b --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/slice.hpp @@ -0,0 +1,189 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +#include <cassert> +#include <iterator> +#include <memory> + +#include <db.h> + +namespace ftcxx { + + class Slice { + public: + Slice() + : _data(nullptr), + _size(0) + {} + + explicit Slice(size_t sz) + : _buf(new char[sz], std::default_delete<char[]>()), + _data(_buf.get()), + _size(sz) + {} + + Slice(const char *p, size_t sz) + : _data(p), + _size(sz) + {} + + explicit Slice(const DBT &d) + : _data(reinterpret_cast<char *>(d.data)), + _size(d.size) + {} + + explicit Slice(const std::string &str) + : _data(str.c_str()), + _size(str.size()) + {} + + Slice(const Slice &other) + : _buf(other._buf), + _data(other._data), + _size(other._size) + {} + + Slice& operator=(const Slice &other) { + _buf = other._buf; + _data = other._data; + _size = other._size; + return *this; + } + + Slice(Slice&& other) + : _buf(), + _data(nullptr), + _size(0) + { + std::swap(_buf, other._buf); + std::swap(_data, other._data); + std::swap(_size, other._size); + } + + Slice& operator=(Slice&& other) { + std::swap(_buf, other._buf); + std::swap(_data, other._data); + std::swap(_size, other._size); + return *this; + } + + template<typename T> + static Slice slice_of(const T &v) { + return Slice(reinterpret_cast<const char *>(&v), sizeof v); + } + + template<typename T> + T as() const { + assert(size() == sizeof(T)); + const T *p = reinterpret_cast<const T *>(data()); + return *p; + } + + const char *data() const { return _data; } + + char *mutable_data() const { + assert(_buf); + return _buf.get(); + } + + size_t size() const { return _size; } + + bool empty() const { return size() == 0; } + + char operator[](size_t n) const { + assert(n < size()); + return _data[n]; + } + + char *begin() { return mutable_data(); } + char *end() { return mutable_data() + size(); } + char *rbegin() { return end(); } + char *rend() { return begin(); } + const char *begin() const { return data(); } + const char *end() const { return data() + size(); } + const char *rbegin() const { return end(); } + const char *rend() const { return begin(); } + const char *cbegin() const { return data(); } + const char *cend() const { return data() + size(); } + const char *crbegin() const { return end(); } + const char *crend() const { return begin(); } + + Slice copy() const { + Slice s(size()); + std::copy(begin(), end(), s.begin()); + return s; + } + + Slice owned() const { + if (_buf) { + return *this; + } else { + return copy(); + } + } + + DBT dbt() const { + DBT d; + d.data = const_cast<void *>(static_cast<const void *>(data())); + d.size = size(); + d.ulen = size(); + d.flags = 0; + return d; + } + + private: + std::shared_ptr<char> _buf; + const char *_data; + size_t _size; + }; + +} // namespace ftcxx + +namespace std { + + template<> + class iterator_traits<ftcxx::Slice> { + typedef typename std::iterator_traits<const char *>::difference_type difference_type; + typedef typename std::iterator_traits<const char *>::value_type value_type; + typedef typename std::iterator_traits<const char *>::pointer pointer; + typedef typename std::iterator_traits<const char *>::reference reference; + typedef typename std::iterator_traits<const char *>::iterator_category iterator_category; + }; + +} // namespace std diff --git a/storage/tokudb/PerconaFT/ftcxx/stats.hpp b/storage/tokudb/PerconaFT/ftcxx/stats.hpp new file mode 100644 index 00000000..5c4cb642 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/stats.hpp @@ -0,0 +1,48 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#pragma once + +namespace ftcxx { + + struct Stats { + Stats() : data_size(0), file_size(0), num_keys(0) {}; + size_t data_size; + size_t file_size; + size_t num_keys; + }; +} diff --git a/storage/tokudb/PerconaFT/ftcxx/tests/CMakeLists.txt b/storage/tokudb/PerconaFT/ftcxx/tests/CMakeLists.txt new file mode 100644 index 00000000..b4db82ff --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/tests/CMakeLists.txt @@ -0,0 +1,47 @@ +include_directories(..) +include_directories(../../src) +include_directories(../../src/tests) + +if (BUILD_TESTING) + find_library(JEMALLOC_STATIC_LIBRARY libjemalloc.a) + + ## reference implementation with simple size-doubling buffer without + ## jemalloc size tricks + add_library(doubling_buffer_ftcxx STATIC + doubling_buffer + ../cursor + ) + add_dependencies(doubling_buffer_ftcxx install_tdb_h) + + foreach (impl + ftcxx + doubling_buffer_ftcxx + ) + foreach (with_jemalloc + ON + OFF + ) + foreach (test + buffer_test + cursor_test + ) + set(_testname ${impl}_${test}) + if (with_jemalloc AND JEMALLOC_STATIC_LIBRARY) + set(_testname ${_testname}_j) + endif () + add_executable(${_testname} ${test}) + if (with_jemalloc AND JEMALLOC_STATIC_LIBRARY) + if (APPLE) + target_link_libraries(${_testname} -Wl,-force_load ${JEMALLOC_STATIC_LIBRARY}) + else () + target_link_libraries(${_testname} -Wl,--whole-archive ${JEMALLOC_STATIC_LIBRARY} -Wl,--no-whole-archive) + endif () + endif () + target_link_libraries(${_testname} ${impl}) + target_link_libraries(${_testname} ${LIBTOKUDB} ${LIBTOKUPORTABILITY}) + + add_test(${_testname} ${_testname}) + endforeach () + endforeach () + endforeach () +endif (BUILD_TESTING)
\ No newline at end of file diff --git a/storage/tokudb/PerconaFT/ftcxx/tests/buffer_test.cpp b/storage/tokudb/PerconaFT/ftcxx/tests/buffer_test.cpp new file mode 100644 index 00000000..293e6443 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/tests/buffer_test.cpp @@ -0,0 +1,217 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <algorithm> +#include <cassert> +#include <iostream> +#include <string> +#include <sstream> +#include <vector> + +#include "buffer.hpp" + +class Item { + const size_t _sz; + +public: + Item(size_t sz=0) + : _sz(sz) + {} + + operator std::string() const { + std::stringstream ss; + ss << "Item(" << _sz << ")"; + return ss.str(); + } + + bool operator==(const Item &other) const { + return _sz == other._sz; + } + + bool operator!=(const Item &other) const { + return !(*this == other); + } + + size_t serialized_size() const { + return (sizeof _sz) + _sz; + } + + void serialize(char *p) const { + size_t *szp = reinterpret_cast<size_t *>(p); + *szp = _sz; + } + + static Item deserialize(const char *p) { + const size_t *szp = reinterpret_cast<const size_t *>(p); + return Item(*szp); + } + + bool check_serialized(const char *p) { + return deserialize(p) == *this; + } +}; + +class SingleSizeGenerator { + const size_t _sz; + +public: + SingleSizeGenerator(size_t sz) + : _sz(sz) + {} + + std::string name() const { + std::stringstream ss; + ss << "SingleSizeGenerator(" << _sz << ")"; + return ss.str(); + } + + Item next() { + return Item(_sz); + } +}; + +class RoundRobinGenerator { + const std::vector<size_t> _szs; + std::vector<size_t>::const_iterator _it; + +public: + RoundRobinGenerator(const std::vector<size_t> &szs) + : _szs(szs), + _it(_szs.begin()) + {} + + std::string name() const { + std::stringstream ss; + ss << "RoundRobinGenerator("; + for (auto it = _szs.begin(); it != _szs.end(); ++it) { + if (it != _szs.begin()) { + ss << ", "; + } + ss << *it; + } + ss << ")"; + return ss.str(); + } + + Item next() { + if (_it == _szs.end()) { + _it = _szs.begin(); + } + return Item(*(_it++)); + } +}; + +template<class Generator> +void test(Generator gen) { + std::vector<Item> expected; + std::vector<Item> received; + + const size_t N = 1000000; + + ftcxx::Buffer b; + + std::cout << gen.name() << ": "; + + for (size_t i = 0; i < N; ++i) { + if (b.full()) { + // drain + while (b.more()) { + Item it = Item::deserialize(b.current()); + received.push_back(it); + b.advance(it.serialized_size()); + } + b.clear(); + } + + // push + Item it = gen.next(); + expected.push_back(it); + char *p = b.alloc(it.serialized_size()); + it.serialize(p); + } + + // drain one more time + while (b.more()) { + Item i = Item::deserialize(b.current()); + received.push_back(i); + b.advance(i.serialized_size()); + } + b.clear(); + + if (expected.size() != received.size()) { + std::cout << "fail" << std::endl; + std::cerr << "expected.size() != received.size()" << std::endl; + std::cerr << expected.size() << " != " << received.size() << std::endl; + return; + } + + for (size_t i = 0; i < expected.size(); ++i) { + if (expected[i] != received[i]) { + std::cout << "fail" << std::endl; + std::cerr << "expected[" << i << "] != received[" << i << "]" << std::endl; + std::cerr << std::string(expected[i]) << " != " << std::string(received[i]) << std::endl; + return; + } + } + + std::cout << "ok" << std::endl; +} + +int main(void) { + test(SingleSizeGenerator(1)); + test(SingleSizeGenerator(3)); + test(SingleSizeGenerator(32)); + test(SingleSizeGenerator(1<<11)); + test(SingleSizeGenerator(1<<12)); + test(SingleSizeGenerator((1<<12) - 1)); + test(SingleSizeGenerator((1<<12) + 1)); + test(SingleSizeGenerator(1<<20)); + + test(RoundRobinGenerator({8, 16})); + test(RoundRobinGenerator({8, 1<<12})); + test(RoundRobinGenerator({8, (1<<12) - 1})); + test(RoundRobinGenerator({8, (1<<12) + 1})); + test(RoundRobinGenerator({8, (1<<12) - 1, (1<<12) + 1})); + test(RoundRobinGenerator({8, (1<<20)})); + test(RoundRobinGenerator({(1<<12) - 1, (1<<12) + 1})); + test(RoundRobinGenerator({(1<<12) , (1<<12) + 1})); + test(RoundRobinGenerator({(1<<12) - 1, (1<<12) })); + test(RoundRobinGenerator({1<<12, 1<<20})); + test(RoundRobinGenerator({1<<16, 1<<17})); + + return 0; +} diff --git a/storage/tokudb/PerconaFT/ftcxx/tests/cursor_test.cpp b/storage/tokudb/PerconaFT/ftcxx/tests/cursor_test.cpp new file mode 100644 index 00000000..5156091f --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/tests/cursor_test.cpp @@ -0,0 +1,178 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <algorithm> +#include <cassert> +#include <cstring> +#include <iostream> +#include <utility> + +#include <db.h> +#include "test.h" // hax + +#include "cursor.hpp" +#include "db.hpp" +#include "db_env.hpp" +#include "db_txn.hpp" +#include "slice.hpp" + +const uint32_t N = 100000; + +static void fill(const ftcxx::DBEnv &env, const ftcxx::DB &db) { + ftcxx::DBTxn txn(env); + + ftcxx::Slice val(1<<10); + memset(val.mutable_data(), 'x', val.size()); + for (uint32_t i = 0; i < N; ++i) { + int r = db.put(txn, ftcxx::Slice::slice_of(i), val); + assert_zero(r); + } + + txn.commit(); +} + +struct UIntComparator { + int operator()(const ftcxx::Slice &a, const ftcxx::Slice &b) { + DBT adbt = a.dbt(); + DBT bdbt = b.dbt(); + return uint_dbt_cmp((DB *) this /*lol*/, &adbt, &bdbt); + } +}; + +static void run_test(const ftcxx::DBEnv &env, const ftcxx::DB &db) { + fill(env, db); + + ftcxx::DBTxn txn(env); + + { + uint32_t lk; + uint32_t rk; + + for (uint32_t i = 0; i < N; i += 1000) { + lk = i; + rk = i + 499; + + ftcxx::Slice key; + ftcxx::Slice val; + uint32_t expect = i; + uint32_t last = 0; + for (auto cur(db.buffered_cursor(txn, ftcxx::Slice::slice_of(lk), ftcxx::Slice::slice_of(rk), + UIntComparator(), ftcxx::DB::NullFilter())); + cur.next(key, val); + ) { + last = key.as<uint32_t>(); + assert(expect == last); + expect++; + } + assert(last == (i + 499)); + } + } + + txn.commit(); + + ftcxx::DBTxn extxn(env); + + { + ftcxx::Slice key; + ftcxx::Slice val; + uint32_t expect = 0; + uint32_t last = 0; + for (auto cur(db.buffered_cursor(extxn, UIntComparator(), ftcxx::DB::NullFilter())); cur.next(key, val); ) { + last = key.as<uint32_t>(); + assert(expect == last); + expect++; + } + assert(last == N - 1); + } + + { + ftcxx::Slice key; + ftcxx::Slice val; + uint32_t expect = 0; + uint32_t last = 0; + for (auto cur(db.simple_cursor(extxn, UIntComparator(), key, val)); ; ) { + std::cout << key.as<uint32_t>() << std::endl; + last = key.as<uint32_t>(); + assert(expect == last); + expect++; + if (!cur.next()) { + break; + } + } + assert(last == N - 1); + } + + extxn.commit(); +} + +int test_main(int argc, char *const argv[]) { + int r; + const char *old_env_dir = TOKU_TEST_FILENAME; + char env_dir[strlen(old_env_dir)+32]; // use unique env directories for parallel tests + snprintf(env_dir, sizeof env_dir, "%s.%d", old_env_dir, getpid()); + const char *db_filename = "ftcxx_cursor_test"; + parse_args(argc, argv); + + char rm_cmd[strlen(env_dir) + strlen("rm -rf ") + 1]; + snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", env_dir); + r = system(rm_cmd); + assert_zero(r); + + r = toku_os_mkdir(env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + assert_zero(r); + + int env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG; + ftcxx::DBEnv env = ftcxx::DBEnvBuilder() + .set_default_bt_compare(uint_dbt_cmp) + .open(env_dir, env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + + ftcxx::DBTxn create_txn(env); + ftcxx::DB db = ftcxx::DBBuilder() + .open(env, create_txn, db_filename, NULL, DB_BTREE, DB_CREATE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + create_txn.commit(); + + run_test(env, db); + + db.close(); + + env.close(); + + r = system(rm_cmd); + assert_zero(r); + + return 0; +} diff --git a/storage/tokudb/PerconaFT/ftcxx/tests/doubling_buffer.cpp b/storage/tokudb/PerconaFT/ftcxx/tests/doubling_buffer.cpp new file mode 100644 index 00000000..e2080180 --- /dev/null +++ b/storage/tokudb/PerconaFT/ftcxx/tests/doubling_buffer.cpp @@ -0,0 +1,118 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include <algorithm> +#include <cassert> +#include <cstdlib> +#include <memory> + +#include "buffer.hpp" + +namespace ftcxx { + + const size_t Buffer::INITIAL_CAPACITY = 1<<10; + const size_t Buffer::MAXIMUM_CAPACITY = 1<<18; + const double Buffer::FULLNESS_RATIO = 0.9; + + Buffer::Buffer() + : _cur(0), + _end(0), + _capacity(INITIAL_CAPACITY), + _buf(nullptr, &std::free) + { + init(); + } + + Buffer::Buffer(size_t capacity) + : _end(0), + _capacity(capacity), + _buf(nullptr, &std::free) + { + init(); + } + + char *Buffer::alloc(size_t sz) { + grow(sz); + char *p = raw(_end); + _end += sz; + return p; + } + + bool Buffer::full() const { + return _end > MAXIMUM_CAPACITY * FULLNESS_RATIO; + } + + bool Buffer::more() const { + return _cur < _end; + } + + char *Buffer::current() const { + return raw(_cur); + } + + void Buffer::advance(size_t sz) { + _cur += sz; + } + + void Buffer::clear() { + _cur = 0; + _end = 0; + } + + void Buffer::init() { + _buf.reset(static_cast<char *>(std::malloc(_capacity))); + } + + size_t Buffer::next_alloc_size(size_t sz) { + return sz * 2; + } + + void Buffer::grow(size_t sz) { + size_t new_capacity = _capacity; + while (new_capacity < _end + sz) { + new_capacity = next_alloc_size(new_capacity); + } + assert(new_capacity >= _capacity); // overflow? + if (new_capacity > _capacity) { + std::unique_ptr<char, void (*)(void *)> new_buf(static_cast<char *>(std::malloc(new_capacity)), &std::free); + std::copy(raw(0), raw(_end), &new_buf.get()[0]); + std::swap(_buf, new_buf); + _capacity = new_capacity; + } + } + +} // namespace ftcxx |