diff options
Diffstat (limited to '')
5 files changed, 934 insertions, 0 deletions
diff --git a/src/rocksdb/utilities/merge_operators/string_append/stringappend.cc b/src/rocksdb/utilities/merge_operators/string_append/stringappend.cc new file mode 100644 index 000000000..5092cabcb --- /dev/null +++ b/src/rocksdb/utilities/merge_operators/string_append/stringappend.cc @@ -0,0 +1,78 @@ +/** + * A MergeOperator for rocksdb that implements string append. + * @author Deon Nicholas (dnicholas@fb.com) + * Copyright 2013 Facebook + */ + +#include "stringappend.h" + +#include <assert.h> + +#include <memory> + +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/options_type.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +static std::unordered_map<std::string, OptionTypeInfo> + stringappend_merge_type_info = { +#ifndef ROCKSDB_LITE + {"delimiter", + {0, OptionType::kString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; +} // namespace +// Constructor: also specify the delimiter character. +StringAppendOperator::StringAppendOperator(char delim_char) + : delim_(1, delim_char) { + RegisterOptions("Delimiter", &delim_, &stringappend_merge_type_info); +} + +StringAppendOperator::StringAppendOperator(const std::string& delim) + : delim_(delim) { + RegisterOptions("Delimiter", &delim_, &stringappend_merge_type_info); +} + +// Implementation for the merge operation (concatenates two strings) +bool StringAppendOperator::Merge(const Slice& /*key*/, + const Slice* existing_value, + const Slice& value, std::string* new_value, + Logger* /*logger*/) const { + // Clear the *new_value for writing. + assert(new_value); + new_value->clear(); + + if (!existing_value) { + // No existing_value. Set *new_value = value + new_value->assign(value.data(), value.size()); + } else { + // Generic append (existing_value != null). + // Reserve *new_value to correct size, and apply concatenation. + new_value->reserve(existing_value->size() + delim_.size() + value.size()); + new_value->assign(existing_value->data(), existing_value->size()); + new_value->append(delim_); + new_value->append(value.data(), value.size()); + } + + return true; +} + +std::shared_ptr<MergeOperator> MergeOperators::CreateStringAppendOperator() { + return std::make_shared<StringAppendOperator>(','); +} + +std::shared_ptr<MergeOperator> MergeOperators::CreateStringAppendOperator( + char delim_char) { + return std::make_shared<StringAppendOperator>(delim_char); +} + +std::shared_ptr<MergeOperator> MergeOperators::CreateStringAppendOperator( + const std::string& delim) { + return std::make_shared<StringAppendOperator>(delim); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/src/rocksdb/utilities/merge_operators/string_append/stringappend.h b/src/rocksdb/utilities/merge_operators/string_append/stringappend.h new file mode 100644 index 000000000..153532382 --- /dev/null +++ b/src/rocksdb/utilities/merge_operators/string_append/stringappend.h @@ -0,0 +1,32 @@ +/** + * A MergeOperator for rocksdb that implements string append. + * @author Deon Nicholas (dnicholas@fb.com) + * Copyright 2013 Facebook + */ + +#pragma once +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +class StringAppendOperator : public AssociativeMergeOperator { + public: + // Constructor: specify delimiter + explicit StringAppendOperator(char delim_char); + explicit StringAppendOperator(const std::string& delim); + + virtual bool Merge(const Slice& key, const Slice* existing_value, + const Slice& value, std::string* new_value, + Logger* logger) const override; + + static const char* kClassName() { return "StringAppendOperator"; } + static const char* kNickName() { return "stringappend"; } + virtual const char* Name() const override { return kClassName(); } + virtual const char* NickName() const override { return kNickName(); } + + private: + std::string delim_; // The delimiter is inserted between elements +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/src/rocksdb/utilities/merge_operators/string_append/stringappend2.cc b/src/rocksdb/utilities/merge_operators/string_append/stringappend2.cc new file mode 100644 index 000000000..36cb9ee34 --- /dev/null +++ b/src/rocksdb/utilities/merge_operators/string_append/stringappend2.cc @@ -0,0 +1,132 @@ +/** + * @author Deon Nicholas (dnicholas@fb.com) + * Copyright 2013 Facebook + */ + +#include "stringappend2.h" + +#include <assert.h> + +#include <memory> +#include <string> + +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" +#include "rocksdb/utilities/options_type.h" +#include "utilities/merge_operators.h" + +namespace ROCKSDB_NAMESPACE { +namespace { +static std::unordered_map<std::string, OptionTypeInfo> + stringappend2_merge_type_info = { +#ifndef ROCKSDB_LITE + {"delimiter", + {0, OptionType::kString, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; +} // namespace + +// Constructor: also specify the delimiter character. +StringAppendTESTOperator::StringAppendTESTOperator(char delim_char) + : delim_(1, delim_char) { + RegisterOptions("Delimiter", &delim_, &stringappend2_merge_type_info); +} + +StringAppendTESTOperator::StringAppendTESTOperator(const std::string& delim) + : delim_(delim) { + RegisterOptions("Delimiter", &delim_, &stringappend2_merge_type_info); +} + +// Implementation for the merge operation (concatenates two strings) +bool StringAppendTESTOperator::FullMergeV2( + const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const { + // Clear the *new_value for writing. + merge_out->new_value.clear(); + + if (merge_in.existing_value == nullptr && merge_in.operand_list.size() == 1) { + // Only one operand + merge_out->existing_operand = merge_in.operand_list.back(); + return true; + } + + // Compute the space needed for the final result. + size_t numBytes = 0; + + for (auto it = merge_in.operand_list.begin(); + it != merge_in.operand_list.end(); ++it) { + numBytes += it->size() + delim_.size(); + } + + // Only print the delimiter after the first entry has been printed + bool printDelim = false; + + // Prepend the *existing_value if one exists. + if (merge_in.existing_value) { + merge_out->new_value.reserve(numBytes + merge_in.existing_value->size()); + merge_out->new_value.append(merge_in.existing_value->data(), + merge_in.existing_value->size()); + printDelim = true; + } else if (numBytes) { + // Without the existing (initial) value, the delimiter before the first of + // subsequent operands becomes redundant. + merge_out->new_value.reserve(numBytes - delim_.size()); + } + + // Concatenate the sequence of strings (and add a delimiter between each) + for (auto it = merge_in.operand_list.begin(); + it != merge_in.operand_list.end(); ++it) { + if (printDelim) { + merge_out->new_value.append(delim_); + } + merge_out->new_value.append(it->data(), it->size()); + printDelim = true; + } + + return true; +} + +bool StringAppendTESTOperator::PartialMergeMulti( + const Slice& /*key*/, const std::deque<Slice>& /*operand_list*/, + std::string* /*new_value*/, Logger* /*logger*/) const { + return false; +} + +// A version of PartialMerge that actually performs "partial merging". +// Use this to simulate the exact behaviour of the StringAppendOperator. +bool StringAppendTESTOperator::_AssocPartialMergeMulti( + const Slice& /*key*/, const std::deque<Slice>& operand_list, + std::string* new_value, Logger* /*logger*/) const { + // Clear the *new_value for writing + assert(new_value); + new_value->clear(); + assert(operand_list.size() >= 2); + + // Generic append + // Determine and reserve correct size for *new_value. + size_t size = 0; + for (const auto& operand : operand_list) { + size += operand.size(); + } + size += (operand_list.size() - 1) * delim_.length(); // Delimiters + new_value->reserve(size); + + // Apply concatenation + new_value->assign(operand_list.front().data(), operand_list.front().size()); + + for (std::deque<Slice>::const_iterator it = operand_list.begin() + 1; + it != operand_list.end(); ++it) { + new_value->append(delim_); + new_value->append(it->data(), it->size()); + } + + return true; +} + +std::shared_ptr<MergeOperator> +MergeOperators::CreateStringAppendTESTOperator() { + return std::make_shared<StringAppendTESTOperator>(','); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/src/rocksdb/utilities/merge_operators/string_append/stringappend2.h b/src/rocksdb/utilities/merge_operators/string_append/stringappend2.h new file mode 100644 index 000000000..75389e4ae --- /dev/null +++ b/src/rocksdb/utilities/merge_operators/string_append/stringappend2.h @@ -0,0 +1,52 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +/** + * A TEST MergeOperator for rocksdb that implements string append. + * It is built using the MergeOperator interface rather than the simpler + * AssociativeMergeOperator interface. This is useful for testing/benchmarking. + * While the two operators are semantically the same, all production code + * should use the StringAppendOperator defined in stringappend.{h,cc}. The + * operator defined in the present file is primarily for testing. + * + * @author Deon Nicholas (dnicholas@fb.com) + * Copyright 2013 Facebook + */ + +#pragma once +#include <deque> +#include <string> + +#include "rocksdb/merge_operator.h" +#include "rocksdb/slice.h" + +namespace ROCKSDB_NAMESPACE { + +class StringAppendTESTOperator : public MergeOperator { + public: + // Constructor with delimiter + explicit StringAppendTESTOperator(char delim_char); + explicit StringAppendTESTOperator(const std::string& delim); + + virtual bool FullMergeV2(const MergeOperationInput& merge_in, + MergeOperationOutput* merge_out) const override; + + virtual bool PartialMergeMulti(const Slice& key, + const std::deque<Slice>& operand_list, + std::string* new_value, + Logger* logger) const override; + + static const char* kClassName() { return "StringAppendTESTOperator"; } + static const char* kNickName() { return "stringappendtest"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kNickName(); } + + private: + // A version of PartialMerge that actually performs "partial merging". + // Use this to simulate the exact behaviour of the StringAppendOperator. + bool _AssocPartialMergeMulti(const Slice& key, + const std::deque<Slice>& operand_list, + std::string* new_value, Logger* logger) const; + + std::string delim_; // The delimiter is inserted between elements +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/src/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc b/src/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc new file mode 100644 index 000000000..22b6144af --- /dev/null +++ b/src/rocksdb/utilities/merge_operators/string_append/stringappend_test.cc @@ -0,0 +1,640 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +/** + * An persistent map : key -> (list of strings), using rocksdb merge. + * This file is a test-harness / use-case for the StringAppendOperator. + * + * @author Deon Nicholas (dnicholas@fb.com) + * Copyright 2013 Facebook, Inc. + */ + +#include "utilities/merge_operators/string_append/stringappend.h" + +#include <iostream> +#include <map> +#include <tuple> + +#include "port/stack_trace.h" +#include "rocksdb/db.h" +#include "rocksdb/merge_operator.h" +#include "rocksdb/utilities/db_ttl.h" +#include "test_util/testharness.h" +#include "util/random.h" +#include "utilities/merge_operators.h" +#include "utilities/merge_operators/string_append/stringappend2.h" + +namespace ROCKSDB_NAMESPACE { + +// Path to the database on file system +const std::string kDbName = test::PerThreadDBPath("stringappend_test"); + +namespace { +// OpenDb opens a (possibly new) rocksdb database with a StringAppendOperator +std::shared_ptr<DB> OpenNormalDb(const std::string& delim) { + DB* db; + Options options; + options.create_if_missing = true; + MergeOperator* mergeOperator; + if (delim.size() == 1) { + mergeOperator = new StringAppendOperator(delim[0]); + } else { + mergeOperator = new StringAppendOperator(delim); + } + options.merge_operator.reset(mergeOperator); + EXPECT_OK(DB::Open(options, kDbName, &db)); + return std::shared_ptr<DB>(db); +} + +#ifndef ROCKSDB_LITE // TtlDb is not supported in Lite +// Open a TtlDB with a non-associative StringAppendTESTOperator +std::shared_ptr<DB> OpenTtlDb(const std::string& delim) { + DBWithTTL* db; + Options options; + options.create_if_missing = true; + MergeOperator* mergeOperator; + if (delim.size() == 1) { + mergeOperator = new StringAppendTESTOperator(delim[0]); + } else { + mergeOperator = new StringAppendTESTOperator(delim); + } + options.merge_operator.reset(mergeOperator); + EXPECT_OK(DBWithTTL::Open(options, kDbName, &db, 123456)); + return std::shared_ptr<DB>(db); +} +#endif // !ROCKSDB_LITE +} // namespace + +/// StringLists represents a set of string-lists, each with a key-index. +/// Supports Append(list, string) and Get(list) +class StringLists { + public: + // Constructor: specifies the rocksdb db + /* implicit */ + StringLists(std::shared_ptr<DB> db) + : db_(db), merge_option_(), get_option_() { + assert(db); + } + + // Append string val onto the list defined by key; return true on success + bool Append(const std::string& key, const std::string& val) { + Slice valSlice(val.data(), val.size()); + auto s = db_->Merge(merge_option_, key, valSlice); + + if (s.ok()) { + return true; + } else { + std::cerr << "ERROR " << s.ToString() << std::endl; + return false; + } + } + + // Returns the list of strings associated with key (or "" if does not exist) + bool Get(const std::string& key, std::string* const result) { + assert(result != nullptr); // we should have a place to store the result + auto s = db_->Get(get_option_, key, result); + + if (s.ok()) { + return true; + } + + // Either key does not exist, or there is some error. + *result = ""; // Always return empty string (just for convention) + + // NotFound is okay; just return empty (similar to std::map) + // But network or db errors, etc, should fail the test (or at least yell) + if (!s.IsNotFound()) { + std::cerr << "ERROR " << s.ToString() << std::endl; + } + + // Always return false if s.ok() was not true + return false; + } + + private: + std::shared_ptr<DB> db_; + WriteOptions merge_option_; + ReadOptions get_option_; +}; + +// The class for unit-testing +class StringAppendOperatorTest : public testing::Test, + public ::testing::WithParamInterface<bool> { + public: + StringAppendOperatorTest() { + EXPECT_OK( + DestroyDB(kDbName, Options())); // Start each test with a fresh DB + } + + void SetUp() override { +#ifndef ROCKSDB_LITE // TtlDb is not supported in Lite + bool if_use_ttl = GetParam(); + if (if_use_ttl) { + fprintf(stderr, "Running tests with ttl db and generic operator.\n"); + StringAppendOperatorTest::SetOpenDbFunction(&OpenTtlDb); + return; + } +#endif // !ROCKSDB_LITE + fprintf(stderr, "Running tests with regular db and operator.\n"); + StringAppendOperatorTest::SetOpenDbFunction(&OpenNormalDb); + } + + using OpenFuncPtr = std::shared_ptr<DB> (*)(const std::string&); + + // Allows user to open databases with different configurations. + // e.g.: Can open a DB or a TtlDB, etc. + static void SetOpenDbFunction(OpenFuncPtr func) { OpenDb = func; } + + protected: + static OpenFuncPtr OpenDb; +}; +StringAppendOperatorTest::OpenFuncPtr StringAppendOperatorTest::OpenDb = + nullptr; + +// THE TEST CASES BEGIN HERE + +TEST_P(StringAppendOperatorTest, IteratorTest) { + auto db_ = OpenDb(","); + StringLists slists(db_); + + slists.Append("k1", "v1"); + slists.Append("k1", "v2"); + slists.Append("k1", "v3"); + + slists.Append("k2", "a1"); + slists.Append("k2", "a2"); + slists.Append("k2", "a3"); + + std::string res; + std::unique_ptr<ROCKSDB_NAMESPACE::Iterator> it( + db_->NewIterator(ReadOptions())); + std::string k1("k1"); + std::string k2("k2"); + bool first = true; + for (it->Seek(k1); it->Valid(); it->Next()) { + res = it->value().ToString(); + if (first) { + ASSERT_EQ(res, "v1,v2,v3"); + first = false; + } else { + ASSERT_EQ(res, "a1,a2,a3"); + } + } + slists.Append("k2", "a4"); + slists.Append("k1", "v4"); + + // Snapshot should still be the same. Should ignore a4 and v4. + first = true; + for (it->Seek(k1); it->Valid(); it->Next()) { + res = it->value().ToString(); + if (first) { + ASSERT_EQ(res, "v1,v2,v3"); + first = false; + } else { + ASSERT_EQ(res, "a1,a2,a3"); + } + } + + // Should release the snapshot and be aware of the new stuff now + it.reset(db_->NewIterator(ReadOptions())); + first = true; + for (it->Seek(k1); it->Valid(); it->Next()) { + res = it->value().ToString(); + if (first) { + ASSERT_EQ(res, "v1,v2,v3,v4"); + first = false; + } else { + ASSERT_EQ(res, "a1,a2,a3,a4"); + } + } + + // start from k2 this time. + for (it->Seek(k2); it->Valid(); it->Next()) { + res = it->value().ToString(); + if (first) { + ASSERT_EQ(res, "v1,v2,v3,v4"); + first = false; + } else { + ASSERT_EQ(res, "a1,a2,a3,a4"); + } + } + + slists.Append("k3", "g1"); + + it.reset(db_->NewIterator(ReadOptions())); + first = true; + std::string k3("k3"); + for (it->Seek(k2); it->Valid(); it->Next()) { + res = it->value().ToString(); + if (first) { + ASSERT_EQ(res, "a1,a2,a3,a4"); + first = false; + } else { + ASSERT_EQ(res, "g1"); + } + } + for (it->Seek(k3); it->Valid(); it->Next()) { + res = it->value().ToString(); + if (first) { + // should not be hit + ASSERT_EQ(res, "a1,a2,a3,a4"); + first = false; + } else { + ASSERT_EQ(res, "g1"); + } + } +} + +TEST_P(StringAppendOperatorTest, SimpleTest) { + auto db = OpenDb(","); + StringLists slists(db); + + slists.Append("k1", "v1"); + slists.Append("k1", "v2"); + slists.Append("k1", "v3"); + + std::string res; + ASSERT_TRUE(slists.Get("k1", &res)); + ASSERT_EQ(res, "v1,v2,v3"); +} + +TEST_P(StringAppendOperatorTest, SimpleDelimiterTest) { + auto db = OpenDb("|"); + StringLists slists(db); + + slists.Append("k1", "v1"); + slists.Append("k1", "v2"); + slists.Append("k1", "v3"); + + std::string res; + ASSERT_TRUE(slists.Get("k1", &res)); + ASSERT_EQ(res, "v1|v2|v3"); +} + +TEST_P(StringAppendOperatorTest, EmptyDelimiterTest) { + auto db = OpenDb(""); + StringLists slists(db); + + slists.Append("k1", "v1"); + slists.Append("k1", "v2"); + slists.Append("k1", "v3"); + + std::string res; + ASSERT_TRUE(slists.Get("k1", &res)); + ASSERT_EQ(res, "v1v2v3"); +} + +TEST_P(StringAppendOperatorTest, MultiCharDelimiterTest) { + auto db = OpenDb("<>"); + StringLists slists(db); + + slists.Append("k1", "v1"); + slists.Append("k1", "v2"); + slists.Append("k1", "v3"); + + std::string res; + ASSERT_TRUE(slists.Get("k1", &res)); + ASSERT_EQ(res, "v1<>v2<>v3"); +} + +TEST_P(StringAppendOperatorTest, DelimiterIsDefensivelyCopiedTest) { + std::string delimiter = "<>"; + auto db = OpenDb(delimiter); + StringLists slists(db); + + slists.Append("k1", "v1"); + slists.Append("k1", "v2"); + delimiter.clear(); + slists.Append("k1", "v3"); + + std::string res; + ASSERT_TRUE(slists.Get("k1", &res)); + ASSERT_EQ(res, "v1<>v2<>v3"); +} + +TEST_P(StringAppendOperatorTest, OneValueNoDelimiterTest) { + auto db = OpenDb("!"); + StringLists slists(db); + + slists.Append("random_key", "single_val"); + + std::string res; + ASSERT_TRUE(slists.Get("random_key", &res)); + ASSERT_EQ(res, "single_val"); +} + +TEST_P(StringAppendOperatorTest, VariousKeys) { + auto db = OpenDb("\n"); + StringLists slists(db); + + slists.Append("c", "asdasd"); + slists.Append("a", "x"); + slists.Append("b", "y"); + slists.Append("a", "t"); + slists.Append("a", "r"); + slists.Append("b", "2"); + slists.Append("c", "asdasd"); + + std::string a, b, c; + bool sa, sb, sc; + sa = slists.Get("a", &a); + sb = slists.Get("b", &b); + sc = slists.Get("c", &c); + + ASSERT_TRUE(sa && sb && sc); // All three keys should have been found + + ASSERT_EQ(a, "x\nt\nr"); + ASSERT_EQ(b, "y\n2"); + ASSERT_EQ(c, "asdasd\nasdasd"); +} + +// Generate semi random keys/words from a small distribution. +TEST_P(StringAppendOperatorTest, RandomMixGetAppend) { + auto db = OpenDb(" "); + StringLists slists(db); + + // Generate a list of random keys and values + const int kWordCount = 15; + std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", + "342839", "dsuha", "mabuais", "sadajsid", + "jf9834hf", "2d9j89", "dj9823jd", "a", + "dk02ed2dh", "$(jd4h984$(*", "mabz"}; + const int kKeyCount = 6; + std::string keys[] = {"dhaiusdhu", "denidw", "daisda", + "keykey", "muki", "shzassdianmd"}; + + // Will store a local copy of all data in order to verify correctness + std::map<std::string, std::string> parallel_copy; + + // Generate a bunch of random queries (Append and Get)! + enum query_t { APPEND_OP, GET_OP, NUM_OPS }; + Random randomGen(1337); // deterministic seed; always get same results! + + const int kNumQueries = 30; + for (int q = 0; q < kNumQueries; ++q) { + // Generate a random query (Append or Get) and random parameters + query_t query = (query_t)randomGen.Uniform((int)NUM_OPS); + std::string key = keys[randomGen.Uniform((int)kKeyCount)]; + std::string word = words[randomGen.Uniform((int)kWordCount)]; + + // Apply the query and any checks. + if (query == APPEND_OP) { + // Apply the rocksdb test-harness Append defined above + slists.Append(key, word); // apply the rocksdb append + + // Apply the similar "Append" to the parallel copy + if (parallel_copy[key].size() > 0) { + parallel_copy[key] += " " + word; + } else { + parallel_copy[key] = word; + } + + } else if (query == GET_OP) { + // Assumes that a non-existent key just returns <empty> + std::string res; + slists.Get(key, &res); + ASSERT_EQ(res, parallel_copy[key]); + } + } +} + +TEST_P(StringAppendOperatorTest, BIGRandomMixGetAppend) { + auto db = OpenDb(" "); + StringLists slists(db); + + // Generate a list of random keys and values + const int kWordCount = 15; + std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", + "342839", "dsuha", "mabuais", "sadajsid", + "jf9834hf", "2d9j89", "dj9823jd", "a", + "dk02ed2dh", "$(jd4h984$(*", "mabz"}; + const int kKeyCount = 6; + std::string keys[] = {"dhaiusdhu", "denidw", "daisda", + "keykey", "muki", "shzassdianmd"}; + + // Will store a local copy of all data in order to verify correctness + std::map<std::string, std::string> parallel_copy; + + // Generate a bunch of random queries (Append and Get)! + enum query_t { APPEND_OP, GET_OP, NUM_OPS }; + Random randomGen(9138204); // deterministic seed + + const int kNumQueries = 1000; + for (int q = 0; q < kNumQueries; ++q) { + // Generate a random query (Append or Get) and random parameters + query_t query = (query_t)randomGen.Uniform((int)NUM_OPS); + std::string key = keys[randomGen.Uniform((int)kKeyCount)]; + std::string word = words[randomGen.Uniform((int)kWordCount)]; + + // Apply the query and any checks. + if (query == APPEND_OP) { + // Apply the rocksdb test-harness Append defined above + slists.Append(key, word); // apply the rocksdb append + + // Apply the similar "Append" to the parallel copy + if (parallel_copy[key].size() > 0) { + parallel_copy[key] += " " + word; + } else { + parallel_copy[key] = word; + } + + } else if (query == GET_OP) { + // Assumes that a non-existent key just returns <empty> + std::string res; + slists.Get(key, &res); + ASSERT_EQ(res, parallel_copy[key]); + } + } +} + +TEST_P(StringAppendOperatorTest, PersistentVariousKeys) { + // Perform the following operations in limited scope + { + auto db = OpenDb("\n"); + StringLists slists(db); + + slists.Append("c", "asdasd"); + slists.Append("a", "x"); + slists.Append("b", "y"); + slists.Append("a", "t"); + slists.Append("a", "r"); + slists.Append("b", "2"); + slists.Append("c", "asdasd"); + + std::string a, b, c; + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_TRUE(slists.Get("c", &c)); + + ASSERT_EQ(a, "x\nt\nr"); + ASSERT_EQ(b, "y\n2"); + ASSERT_EQ(c, "asdasd\nasdasd"); + } + + // Reopen the database (the previous changes should persist / be remembered) + { + auto db = OpenDb("\n"); + StringLists slists(db); + + slists.Append("c", "bbnagnagsx"); + slists.Append("a", "sa"); + slists.Append("b", "df"); + slists.Append("a", "gh"); + slists.Append("a", "jk"); + slists.Append("b", "l;"); + slists.Append("c", "rogosh"); + + // The previous changes should be on disk (L0) + // The most recent changes should be in memory (MemTable) + // Hence, this will test both Get() paths. + std::string a, b, c; + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_TRUE(slists.Get("c", &c)); + + ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk"); + ASSERT_EQ(b, "y\n2\ndf\nl;"); + ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh"); + } + + // Reopen the database (the previous changes should persist / be remembered) + { + auto db = OpenDb("\n"); + StringLists slists(db); + + // All changes should be on disk. This will test VersionSet Get() + std::string a, b, c; + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_TRUE(slists.Get("c", &c)); + + ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk"); + ASSERT_EQ(b, "y\n2\ndf\nl;"); + ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh"); + } +} + +TEST_P(StringAppendOperatorTest, PersistentFlushAndCompaction) { + // Perform the following operations in limited scope + { + auto db = OpenDb("\n"); + StringLists slists(db); + std::string a, b, c; + + // Append, Flush, Get + slists.Append("c", "asdasd"); + ASSERT_OK(db->Flush(ROCKSDB_NAMESPACE::FlushOptions())); + ASSERT_TRUE(slists.Get("c", &c)); + ASSERT_EQ(c, "asdasd"); + + // Append, Flush, Append, Get + slists.Append("a", "x"); + slists.Append("b", "y"); + ASSERT_OK(db->Flush(ROCKSDB_NAMESPACE::FlushOptions())); + slists.Append("a", "t"); + slists.Append("a", "r"); + slists.Append("b", "2"); + + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_EQ(a, "x\nt\nr"); + + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_EQ(b, "y\n2"); + + // Append, Get + ASSERT_TRUE(slists.Append("c", "asdasd")); + ASSERT_TRUE(slists.Append("b", "monkey")); + + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_TRUE(slists.Get("c", &c)); + + ASSERT_EQ(a, "x\nt\nr"); + ASSERT_EQ(b, "y\n2\nmonkey"); + ASSERT_EQ(c, "asdasd\nasdasd"); + } + + // Reopen the database (the previous changes should persist / be remembered) + { + auto db = OpenDb("\n"); + StringLists slists(db); + std::string a, b, c; + + // Get (Quick check for persistence of previous database) + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_EQ(a, "x\nt\nr"); + + // Append, Compact, Get + slists.Append("c", "bbnagnagsx"); + slists.Append("a", "sa"); + slists.Append("b", "df"); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_TRUE(slists.Get("c", &c)); + ASSERT_EQ(a, "x\nt\nr\nsa"); + ASSERT_EQ(b, "y\n2\nmonkey\ndf"); + ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx"); + + // Append, Get + slists.Append("a", "gh"); + slists.Append("a", "jk"); + slists.Append("b", "l;"); + slists.Append("c", "rogosh"); + ASSERT_TRUE(slists.Get("a", &a)); + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_TRUE(slists.Get("c", &c)); + ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk"); + ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;"); + ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh"); + + // Compact, Get + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk"); + ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;"); + ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh"); + + // Append, Flush, Compact, Get + slists.Append("b", "afcg"); + ASSERT_OK(db->Flush(ROCKSDB_NAMESPACE::FlushOptions())); + ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_TRUE(slists.Get("b", &b)); + ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;\nafcg"); + } +} + +TEST_P(StringAppendOperatorTest, SimpleTestNullDelimiter) { + auto db = OpenDb(std::string(1, '\0')); + StringLists slists(db); + + slists.Append("k1", "v1"); + slists.Append("k1", "v2"); + slists.Append("k1", "v3"); + + std::string res; + ASSERT_TRUE(slists.Get("k1", &res)); + + // Construct the desired string. Default constructor doesn't like '\0' chars. + std::string checker("v1,v2,v3"); // Verify that the string is right size. + checker[2] = '\0'; // Use null delimiter instead of comma. + checker[5] = '\0'; + ASSERT_EQ(checker.size(), 8); // Verify it is still the correct size + + // Check that the rocksdb result string matches the desired string + ASSERT_EQ(res.size(), checker.size()); + ASSERT_EQ(res, checker); +} + +INSTANTIATE_TEST_CASE_P(StringAppendOperatorTest, StringAppendOperatorTest, + testing::Bool()); + +} // namespace ROCKSDB_NAMESPACE + +int main(int argc, char** argv) { + ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} |