summaryrefslogtreecommitdiffstats
path: root/src/rocksdb/db/compaction/compaction_job_test.cc
diff options
context:
space:
mode:
Diffstat (limited to 'src/rocksdb/db/compaction/compaction_job_test.cc')
-rw-r--r--src/rocksdb/db/compaction/compaction_job_test.cc1082
1 files changed, 1082 insertions, 0 deletions
diff --git a/src/rocksdb/db/compaction/compaction_job_test.cc b/src/rocksdb/db/compaction/compaction_job_test.cc
new file mode 100644
index 000000000..e7b46ef97
--- /dev/null
+++ b/src/rocksdb/db/compaction/compaction_job_test.cc
@@ -0,0 +1,1082 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include <algorithm>
+#include <array>
+#include <cinttypes>
+#include <map>
+#include <string>
+#include <tuple>
+
+#include "db/blob_index.h"
+#include "db/column_family.h"
+#include "db/compaction/compaction_job.h"
+#include "db/db_impl/db_impl.h"
+#include "db/error_handler.h"
+#include "db/version_set.h"
+#include "file/writable_file_writer.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "rocksdb/write_buffer_manager.h"
+#include "table/mock_table.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+#include "util/string_util.h"
+#include "utilities/merge_operators.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+namespace {
+
+void VerifyInitializationOfCompactionJobStats(
+ const CompactionJobStats& compaction_job_stats) {
+#if !defined(IOS_CROSS_COMPILE)
+ ASSERT_EQ(compaction_job_stats.elapsed_micros, 0U);
+
+ ASSERT_EQ(compaction_job_stats.num_input_records, 0U);
+ ASSERT_EQ(compaction_job_stats.num_input_files, 0U);
+ ASSERT_EQ(compaction_job_stats.num_input_files_at_output_level, 0U);
+
+ ASSERT_EQ(compaction_job_stats.num_output_records, 0U);
+ ASSERT_EQ(compaction_job_stats.num_output_files, 0U);
+
+ ASSERT_EQ(compaction_job_stats.is_manual_compaction, true);
+
+ ASSERT_EQ(compaction_job_stats.total_input_bytes, 0U);
+ ASSERT_EQ(compaction_job_stats.total_output_bytes, 0U);
+
+ ASSERT_EQ(compaction_job_stats.total_input_raw_key_bytes, 0U);
+ ASSERT_EQ(compaction_job_stats.total_input_raw_value_bytes, 0U);
+
+ ASSERT_EQ(compaction_job_stats.smallest_output_key_prefix[0], 0);
+ ASSERT_EQ(compaction_job_stats.largest_output_key_prefix[0], 0);
+
+ ASSERT_EQ(compaction_job_stats.num_records_replaced, 0U);
+
+ ASSERT_EQ(compaction_job_stats.num_input_deletion_records, 0U);
+ ASSERT_EQ(compaction_job_stats.num_expired_deletion_records, 0U);
+
+ ASSERT_EQ(compaction_job_stats.num_corrupt_keys, 0U);
+#endif // !defined(IOS_CROSS_COMPILE)
+}
+
+} // namespace
+
+// TODO(icanadi) Make it simpler once we mock out VersionSet
+class CompactionJobTest : public testing::Test {
+ public:
+ CompactionJobTest()
+ : env_(Env::Default()),
+ fs_(std::make_shared<LegacyFileSystemWrapper>(env_)),
+ dbname_(test::PerThreadDBPath("compaction_job_test")),
+ db_options_(),
+ mutable_cf_options_(cf_options_),
+ table_cache_(NewLRUCache(50000, 16)),
+ write_buffer_manager_(db_options_.db_write_buffer_size),
+ versions_(new VersionSet(dbname_, &db_options_, env_options_,
+ table_cache_.get(), &write_buffer_manager_,
+ &write_controller_,
+ /*block_cache_tracer=*/nullptr)),
+ shutting_down_(false),
+ preserve_deletes_seqnum_(0),
+ mock_table_factory_(new mock::MockTableFactory()),
+ error_handler_(nullptr, db_options_, &mutex_) {
+ EXPECT_OK(env_->CreateDirIfMissing(dbname_));
+ db_options_.env = env_;
+ db_options_.fs = fs_;
+ db_options_.db_paths.emplace_back(dbname_,
+ std::numeric_limits<uint64_t>::max());
+ }
+
+ std::string GenerateFileName(uint64_t file_number) {
+ FileMetaData meta;
+ std::vector<DbPath> db_paths;
+ db_paths.emplace_back(dbname_, std::numeric_limits<uint64_t>::max());
+ meta.fd = FileDescriptor(file_number, 0, 0);
+ return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
+ }
+
+ static std::string KeyStr(const std::string& user_key,
+ const SequenceNumber seq_num, const ValueType t) {
+ return InternalKey(user_key, seq_num, t).Encode().ToString();
+ }
+
+ static std::string BlobStr(uint64_t blob_file_number, uint64_t offset,
+ uint64_t size) {
+ std::string blob_index;
+ BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size,
+ kNoCompression);
+ return blob_index;
+ }
+
+ static std::string BlobStrTTL(uint64_t blob_file_number, uint64_t offset,
+ uint64_t size, uint64_t expiration) {
+ std::string blob_index;
+ BlobIndex::EncodeBlobTTL(&blob_index, expiration, blob_file_number, offset,
+ size, kNoCompression);
+ return blob_index;
+ }
+
+ static std::string BlobStrInlinedTTL(const Slice& value,
+ uint64_t expiration) {
+ std::string blob_index;
+ BlobIndex::EncodeInlinedTTL(&blob_index, expiration, value);
+ return blob_index;
+ }
+
+ void AddMockFile(const stl_wrappers::KVMap& contents, int level = 0) {
+ assert(contents.size() > 0);
+
+ bool first_key = true;
+ std::string smallest, largest;
+ InternalKey smallest_key, largest_key;
+ SequenceNumber smallest_seqno = kMaxSequenceNumber;
+ SequenceNumber largest_seqno = 0;
+ uint64_t oldest_blob_file_number = kInvalidBlobFileNumber;
+ for (auto kv : contents) {
+ ParsedInternalKey key;
+ std::string skey;
+ std::string value;
+ std::tie(skey, value) = kv;
+ bool parsed = ParseInternalKey(skey, &key);
+
+ smallest_seqno = std::min(smallest_seqno, key.sequence);
+ largest_seqno = std::max(largest_seqno, key.sequence);
+
+ if (first_key ||
+ cfd_->user_comparator()->Compare(key.user_key, smallest) < 0) {
+ smallest.assign(key.user_key.data(), key.user_key.size());
+ smallest_key.DecodeFrom(skey);
+ }
+ if (first_key ||
+ cfd_->user_comparator()->Compare(key.user_key, largest) > 0) {
+ largest.assign(key.user_key.data(), key.user_key.size());
+ largest_key.DecodeFrom(skey);
+ }
+
+ first_key = false;
+
+ if (parsed && key.type == kTypeBlobIndex) {
+ BlobIndex blob_index;
+ const Status s = blob_index.DecodeFrom(value);
+ if (!s.ok()) {
+ continue;
+ }
+
+ if (blob_index.IsInlined() || blob_index.HasTTL() ||
+ blob_index.file_number() == kInvalidBlobFileNumber) {
+ continue;
+ }
+
+ if (oldest_blob_file_number == kInvalidBlobFileNumber ||
+ oldest_blob_file_number > blob_index.file_number()) {
+ oldest_blob_file_number = blob_index.file_number();
+ }
+ }
+ }
+
+ uint64_t file_number = versions_->NewFileNumber();
+ EXPECT_OK(mock_table_factory_->CreateMockTable(
+ env_, GenerateFileName(file_number), std::move(contents)));
+
+ VersionEdit edit;
+ edit.AddFile(level, file_number, 0, 10, smallest_key, largest_key,
+ smallest_seqno, largest_seqno, false, oldest_blob_file_number,
+ kUnknownOldestAncesterTime, kUnknownFileCreationTime,
+ kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+
+ mutex_.Lock();
+ versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
+ mutable_cf_options_, &edit, &mutex_);
+ mutex_.Unlock();
+ }
+
+ void SetLastSequence(const SequenceNumber sequence_number) {
+ versions_->SetLastAllocatedSequence(sequence_number + 1);
+ versions_->SetLastPublishedSequence(sequence_number + 1);
+ versions_->SetLastSequence(sequence_number + 1);
+ }
+
+ // returns expected result after compaction
+ stl_wrappers::KVMap CreateTwoFiles(bool gen_corrupted_keys) {
+ auto expected_results = mock::MakeMockFile();
+ const int kKeysPerFile = 10000;
+ const int kCorruptKeysPerFile = 200;
+ const int kMatchingKeys = kKeysPerFile / 2;
+ SequenceNumber sequence_number = 0;
+
+ auto corrupt_id = [&](int id) {
+ return gen_corrupted_keys && id > 0 && id <= kCorruptKeysPerFile;
+ };
+
+ for (int i = 0; i < 2; ++i) {
+ auto contents = mock::MakeMockFile();
+ for (int k = 0; k < kKeysPerFile; ++k) {
+ auto key = ToString(i * kMatchingKeys + k);
+ auto value = ToString(i * kKeysPerFile + k);
+ InternalKey internal_key(key, ++sequence_number, kTypeValue);
+
+ // This is how the key will look like once it's written in bottommost
+ // file
+ InternalKey bottommost_internal_key(
+ key, 0, kTypeValue);
+
+ if (corrupt_id(k)) {
+ test::CorruptKeyType(&internal_key);
+ test::CorruptKeyType(&bottommost_internal_key);
+ }
+ contents.insert({ internal_key.Encode().ToString(), value });
+ if (i == 1 || k < kMatchingKeys || corrupt_id(k - kMatchingKeys)) {
+ expected_results.insert(
+ { bottommost_internal_key.Encode().ToString(), value });
+ }
+ }
+
+ AddMockFile(contents);
+ }
+
+ SetLastSequence(sequence_number);
+
+ return expected_results;
+ }
+
+ void NewDB() {
+ DestroyDB(dbname_, Options());
+ EXPECT_OK(env_->CreateDirIfMissing(dbname_));
+ versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
+ table_cache_.get(), &write_buffer_manager_,
+ &write_controller_,
+ /*block_cache_tracer=*/nullptr));
+ compaction_job_stats_.Reset();
+ SetIdentityFile(env_, dbname_);
+
+ VersionEdit new_db;
+ if (db_options_.write_dbid_to_manifest) {
+ DBImpl* impl = new DBImpl(DBOptions(), dbname_);
+ std::string db_id;
+ impl->GetDbIdentityFromIdentityFile(&db_id);
+ new_db.SetDBId(db_id);
+ }
+ new_db.SetLogNumber(0);
+ new_db.SetNextFile(2);
+ new_db.SetLastSequence(0);
+
+ const std::string manifest = DescriptorFileName(dbname_, 1);
+ std::unique_ptr<WritableFile> file;
+ Status s = env_->NewWritableFile(
+ manifest, &file, env_->OptimizeForManifestWrite(env_options_));
+ ASSERT_OK(s);
+ std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+ NewLegacyWritableFileWrapper(std::move(file)), manifest, env_options_));
+ {
+ log::Writer log(std::move(file_writer), 0, false);
+ std::string record;
+ new_db.EncodeTo(&record);
+ s = log.AddRecord(record);
+ }
+ ASSERT_OK(s);
+ // Make "CURRENT" file that points to the new manifest file.
+ s = SetCurrentFile(env_, dbname_, 1, nullptr);
+
+ std::vector<ColumnFamilyDescriptor> column_families;
+ cf_options_.table_factory = mock_table_factory_;
+ cf_options_.merge_operator = merge_op_;
+ cf_options_.compaction_filter = compaction_filter_.get();
+ column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
+
+ EXPECT_OK(versions_->Recover(column_families, false));
+ cfd_ = versions_->GetColumnFamilySet()->GetDefault();
+ }
+
+ void RunCompaction(
+ const std::vector<std::vector<FileMetaData*>>& input_files,
+ const stl_wrappers::KVMap& expected_results,
+ const std::vector<SequenceNumber>& snapshots = {},
+ SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber,
+ int output_level = 1, bool verify = true,
+ uint64_t expected_oldest_blob_file_number = kInvalidBlobFileNumber) {
+ auto cfd = versions_->GetColumnFamilySet()->GetDefault();
+
+ size_t num_input_files = 0;
+ std::vector<CompactionInputFiles> compaction_input_files;
+ for (size_t level = 0; level < input_files.size(); level++) {
+ auto level_files = input_files[level];
+ CompactionInputFiles compaction_level;
+ compaction_level.level = static_cast<int>(level);
+ compaction_level.files.insert(compaction_level.files.end(),
+ level_files.begin(), level_files.end());
+ compaction_input_files.push_back(compaction_level);
+ num_input_files += level_files.size();
+ }
+
+ Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(),
+ *cfd->GetLatestMutableCFOptions(),
+ compaction_input_files, output_level, 1024 * 1024,
+ 10 * 1024 * 1024, 0, kNoCompression,
+ cfd->ioptions()->compression_opts, 0, {}, true);
+ compaction.SetInputVersion(cfd->current());
+
+ LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
+ mutex_.Lock();
+ EventLogger event_logger(db_options_.info_log.get());
+ // TODO(yiwu) add a mock snapshot checker and add test for it.
+ SnapshotChecker* snapshot_checker = nullptr;
+ CompactionJob compaction_job(
+ 0, &compaction, db_options_, env_options_, versions_.get(),
+ &shutting_down_, preserve_deletes_seqnum_, &log_buffer, nullptr,
+ nullptr, nullptr, &mutex_, &error_handler_, snapshots,
+ earliest_write_conflict_snapshot, snapshot_checker, table_cache_,
+ &event_logger, false, false, dbname_, &compaction_job_stats_,
+ Env::Priority::USER);
+ VerifyInitializationOfCompactionJobStats(compaction_job_stats_);
+
+ compaction_job.Prepare();
+ mutex_.Unlock();
+ Status s;
+ s = compaction_job.Run();
+ ASSERT_OK(s);
+ mutex_.Lock();
+ ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions()));
+ mutex_.Unlock();
+
+ if (verify) {
+ ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U);
+ ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files);
+
+ if (expected_results.empty()) {
+ ASSERT_EQ(compaction_job_stats_.num_output_files, 0U);
+ } else {
+ ASSERT_EQ(compaction_job_stats_.num_output_files, 1U);
+ mock_table_factory_->AssertLatestFile(expected_results);
+
+ auto output_files =
+ cfd->current()->storage_info()->LevelFiles(output_level);
+ ASSERT_EQ(output_files.size(), 1);
+ ASSERT_EQ(output_files[0]->oldest_blob_file_number,
+ expected_oldest_blob_file_number);
+ }
+ }
+ }
+
+ Env* env_;
+ std::shared_ptr<FileSystem> fs_;
+ std::string dbname_;
+ EnvOptions env_options_;
+ ImmutableDBOptions db_options_;
+ ColumnFamilyOptions cf_options_;
+ MutableCFOptions mutable_cf_options_;
+ std::shared_ptr<Cache> table_cache_;
+ WriteController write_controller_;
+ WriteBufferManager write_buffer_manager_;
+ std::unique_ptr<VersionSet> versions_;
+ InstrumentedMutex mutex_;
+ std::atomic<bool> shutting_down_;
+ SequenceNumber preserve_deletes_seqnum_;
+ std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
+ CompactionJobStats compaction_job_stats_;
+ ColumnFamilyData* cfd_;
+ std::unique_ptr<CompactionFilter> compaction_filter_;
+ std::shared_ptr<MergeOperator> merge_op_;
+ ErrorHandler error_handler_;
+};
+
+TEST_F(CompactionJobTest, Simple) {
+ NewDB();
+
+ auto expected_results = CreateTwoFiles(false);
+ auto cfd = versions_->GetColumnFamilySet()->GetDefault();
+ auto files = cfd->current()->storage_info()->LevelFiles(0);
+ ASSERT_EQ(2U, files.size());
+ RunCompaction({ files }, expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleCorrupted) {
+ NewDB();
+
+ auto expected_results = CreateTwoFiles(true);
+ auto cfd = versions_->GetColumnFamilySet()->GetDefault();
+ auto files = cfd->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+ ASSERT_EQ(compaction_job_stats_.num_corrupt_keys, 400U);
+}
+
+TEST_F(CompactionJobTest, SimpleDeletion) {
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({{KeyStr("c", 4U, kTypeDeletion), ""},
+ {KeyStr("c", 3U, kTypeValue), "val"}});
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({{KeyStr("b", 2U, kTypeValue), "val"},
+ {KeyStr("b", 1U, kTypeValue), "val"}});
+ AddMockFile(file2);
+
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("b", 0U, kTypeValue), "val"}});
+
+ SetLastSequence(4U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, OutputNothing) {
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}});
+
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({{KeyStr("a", 2U, kTypeDeletion), ""}});
+
+ AddMockFile(file2);
+
+ auto expected_results = mock::MakeMockFile();
+
+ SetLastSequence(4U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleOverwrite) {
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("a", 3U, kTypeValue), "val2"},
+ {KeyStr("b", 4U, kTypeValue), "val3"},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
+ {KeyStr("b", 2U, kTypeValue), "val"}});
+ AddMockFile(file2);
+
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "val2"},
+ {KeyStr("b", 0U, kTypeValue), "val3"}});
+
+ SetLastSequence(4U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleNonLastLevel) {
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("a", 5U, kTypeValue), "val2"},
+ {KeyStr("b", 6U, kTypeValue), "val3"},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"},
+ {KeyStr("b", 4U, kTypeValue), "val"}});
+ AddMockFile(file2, 1);
+
+ auto file3 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
+ {KeyStr("b", 2U, kTypeValue), "val"}});
+ AddMockFile(file3, 2);
+
+ // Because level 1 is not the last level, the sequence numbers of a and b
+ // cannot be set to 0
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"},
+ {KeyStr("b", 6U, kTypeValue), "val3"}});
+
+ SetLastSequence(6U);
+ auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
+ auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
+ RunCompaction({lvl0_files, lvl1_files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleMerge) {
+ merge_op_ = MergeOperators::CreateStringAppendOperator();
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("a", 5U, kTypeMerge), "5"},
+ {KeyStr("a", 4U, kTypeMerge), "4"},
+ {KeyStr("a", 3U, kTypeValue), "3"},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile(
+ {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeValue), "1"}});
+ AddMockFile(file2);
+
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
+ {KeyStr("b", 0U, kTypeValue), "1,2"}});
+
+ SetLastSequence(5U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, NonAssocMerge) {
+ merge_op_ = MergeOperators::CreateStringAppendTESTOperator();
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("a", 5U, kTypeMerge), "5"},
+ {KeyStr("a", 4U, kTypeMerge), "4"},
+ {KeyStr("a", 3U, kTypeMerge), "3"},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile(
+ {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeMerge), "1"}});
+ AddMockFile(file2);
+
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
+ {KeyStr("b", 0U, kTypeValue), "1,2"}});
+
+ SetLastSequence(5U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+// Filters merge operands with value 10.
+TEST_F(CompactionJobTest, MergeOperandFilter) {
+ merge_op_ = MergeOperators::CreateUInt64AddOperator();
+ compaction_filter_.reset(new test::FilterNumber(10U));
+ NewDB();
+
+ auto file1 = mock::MakeMockFile(
+ {{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)},
+ {KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)}, // Filtered
+ {KeyStr("a", 3U, kTypeMerge), test::EncodeInt(3U)}});
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({
+ {KeyStr("b", 2U, kTypeMerge), test::EncodeInt(2U)},
+ {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)} // Filtered
+ });
+ AddMockFile(file2);
+
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), test::EncodeInt(8U)},
+ {KeyStr("b", 0U, kTypeValue), test::EncodeInt(2U)}});
+
+ SetLastSequence(5U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, FilterSomeMergeOperands) {
+ merge_op_ = MergeOperators::CreateUInt64AddOperator();
+ compaction_filter_.reset(new test::FilterNumber(10U));
+ NewDB();
+
+ auto file1 = mock::MakeMockFile(
+ {{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)},
+ {KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)}, // Filtered
+ {KeyStr("a", 3U, kTypeValue), test::EncodeInt(5U)},
+ {KeyStr("d", 8U, kTypeMerge), test::EncodeInt(10U)}});
+ AddMockFile(file1);
+
+ auto file2 =
+ mock::MakeMockFile({{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("c", 2U, kTypeMerge), test::EncodeInt(3U)},
+ {KeyStr("c", 1U, kTypeValue), test::EncodeInt(7U)},
+ {KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)}});
+ AddMockFile(file2);
+
+ auto file3 =
+ mock::MakeMockFile({{KeyStr("a", 1U, kTypeMerge), test::EncodeInt(3U)}});
+ AddMockFile(file3, 2);
+
+ auto expected_results = mock::MakeMockFile({
+ {KeyStr("a", 5U, kTypeValue), test::EncodeInt(10U)},
+ {KeyStr("c", 2U, kTypeValue), test::EncodeInt(10U)},
+ {KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)}
+ // b does not appear because the operands are filtered
+ });
+
+ SetLastSequence(5U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+// Test where all operands/merge results are filtered out.
+TEST_F(CompactionJobTest, FilterAllMergeOperands) {
+ merge_op_ = MergeOperators::CreateUInt64AddOperator();
+ compaction_filter_.reset(new test::FilterNumber(10U));
+ NewDB();
+
+ auto file1 =
+ mock::MakeMockFile({{KeyStr("a", 11U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("a", 10U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("a", 9U, kTypeMerge), test::EncodeInt(10U)}});
+ AddMockFile(file1);
+
+ auto file2 =
+ mock::MakeMockFile({{KeyStr("b", 8U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 7U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 6U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 5U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 4U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 3U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("c", 2U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("c", 1U, kTypeMerge), test::EncodeInt(10U)}});
+ AddMockFile(file2);
+
+ auto file3 =
+ mock::MakeMockFile({{KeyStr("a", 2U, kTypeMerge), test::EncodeInt(10U)},
+ {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)}});
+ AddMockFile(file3, 2);
+
+ SetLastSequence(11U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+
+ stl_wrappers::KVMap empty_map;
+ RunCompaction({files}, empty_map);
+}
+
+TEST_F(CompactionJobTest, SimpleSingleDelete) {
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("a", 5U, kTypeDeletion), ""},
+ {KeyStr("b", 6U, kTypeSingleDeletion), ""},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"},
+ {KeyStr("b", 4U, kTypeValue), "val"}});
+ AddMockFile(file2);
+
+ auto file3 = mock::MakeMockFile({
+ {KeyStr("a", 1U, kTypeValue), "val"},
+ });
+ AddMockFile(file3, 2);
+
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("a", 5U, kTypeDeletion), ""}});
+
+ SetLastSequence(6U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, SingleDeleteSnapshots) {
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("A", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("a", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("b", 21U, kTypeSingleDeletion), ""},
+ {KeyStr("c", 22U, kTypeSingleDeletion), ""},
+ {KeyStr("d", 9U, kTypeSingleDeletion), ""},
+ {KeyStr("f", 21U, kTypeSingleDeletion), ""},
+ {KeyStr("j", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("j", 9U, kTypeSingleDeletion), ""},
+ {KeyStr("k", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("k", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("l", 3U, kTypeSingleDeletion), ""},
+ {KeyStr("l", 2U, kTypeSingleDeletion), ""},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({
+ {KeyStr("0", 2U, kTypeSingleDeletion), ""},
+ {KeyStr("a", 11U, kTypeValue), "val1"},
+ {KeyStr("b", 11U, kTypeValue), "val2"},
+ {KeyStr("c", 21U, kTypeValue), "val3"},
+ {KeyStr("d", 8U, kTypeValue), "val4"},
+ {KeyStr("e", 2U, kTypeSingleDeletion), ""},
+ {KeyStr("f", 1U, kTypeValue), "val1"},
+ {KeyStr("g", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("h", 2U, kTypeSingleDeletion), ""},
+ {KeyStr("m", 12U, kTypeValue), "val1"},
+ {KeyStr("m", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("m", 8U, kTypeValue), "val2"},
+ });
+ AddMockFile(file2);
+
+ auto file3 = mock::MakeMockFile({
+ {KeyStr("A", 1U, kTypeValue), "val"},
+ {KeyStr("e", 1U, kTypeValue), "val"},
+ });
+ AddMockFile(file3, 2);
+
+ auto expected_results = mock::MakeMockFile({
+ {KeyStr("A", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("a", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("a", 11U, kTypeValue), ""},
+ {KeyStr("b", 21U, kTypeSingleDeletion), ""},
+ {KeyStr("b", 11U, kTypeValue), "val2"},
+ {KeyStr("c", 22U, kTypeSingleDeletion), ""},
+ {KeyStr("c", 21U, kTypeValue), ""},
+ {KeyStr("e", 2U, kTypeSingleDeletion), ""},
+ {KeyStr("f", 21U, kTypeSingleDeletion), ""},
+ {KeyStr("f", 1U, kTypeValue), "val1"},
+ {KeyStr("g", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("j", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("k", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("m", 12U, kTypeValue), "val1"},
+ {KeyStr("m", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("m", 8U, kTypeValue), "val2"},
+ });
+
+ SetLastSequence(22U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results, {10U, 20U}, 10U);
+}
+
+TEST_F(CompactionJobTest, EarliestWriteConflictSnapshot) {
+ NewDB();
+
+ // Test multiple snapshots where the earliest snapshot is not a
+ // write-conflic-snapshot.
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("A", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("A", 23U, kTypeValue), "val"},
+ {KeyStr("B", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("B", 23U, kTypeValue), "val"},
+ {KeyStr("D", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("G", 32U, kTypeSingleDeletion), ""},
+ {KeyStr("G", 31U, kTypeValue), "val"},
+ {KeyStr("G", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("G", 23U, kTypeValue), "val2"},
+ {KeyStr("H", 31U, kTypeValue), "val"},
+ {KeyStr("H", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("H", 23U, kTypeValue), "val"},
+ {KeyStr("I", 35U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 34U, kTypeValue), "val2"},
+ {KeyStr("I", 33U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 32U, kTypeValue), "val3"},
+ {KeyStr("I", 31U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 34U, kTypeValue), "val"},
+ {KeyStr("J", 33U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 25U, kTypeValue), "val2"},
+ {KeyStr("J", 24U, kTypeSingleDeletion), ""},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({
+ {KeyStr("A", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("A", 13U, kTypeValue), "val2"},
+ {KeyStr("C", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("C", 13U, kTypeValue), "val"},
+ {KeyStr("E", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("F", 4U, kTypeSingleDeletion), ""},
+ {KeyStr("F", 3U, kTypeValue), "val"},
+ {KeyStr("G", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("G", 13U, kTypeValue), "val3"},
+ {KeyStr("H", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("H", 13U, kTypeValue), "val2"},
+ {KeyStr("I", 13U, kTypeValue), "val4"},
+ {KeyStr("I", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 11U, kTypeValue), "val5"},
+ {KeyStr("J", 15U, kTypeValue), "val3"},
+ {KeyStr("J", 14U, kTypeSingleDeletion), ""},
+ });
+ AddMockFile(file2);
+
+ auto expected_results = mock::MakeMockFile({
+ {KeyStr("A", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("A", 23U, kTypeValue), ""},
+ {KeyStr("B", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("B", 23U, kTypeValue), ""},
+ {KeyStr("D", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("E", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("G", 32U, kTypeSingleDeletion), ""},
+ {KeyStr("G", 31U, kTypeValue), ""},
+ {KeyStr("H", 31U, kTypeValue), "val"},
+ {KeyStr("I", 35U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 34U, kTypeValue), ""},
+ {KeyStr("I", 31U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 13U, kTypeValue), "val4"},
+ {KeyStr("J", 34U, kTypeValue), "val"},
+ {KeyStr("J", 33U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 25U, kTypeValue), "val2"},
+ {KeyStr("J", 24U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 15U, kTypeValue), "val3"},
+ {KeyStr("J", 14U, kTypeSingleDeletion), ""},
+ });
+
+ SetLastSequence(24U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results, {10U, 20U, 30U}, 20U);
+}
+
+TEST_F(CompactionJobTest, SingleDeleteZeroSeq) {
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("A", 10U, kTypeSingleDeletion), ""},
+ {KeyStr("dummy", 5U, kTypeValue), "val2"},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({
+ {KeyStr("A", 0U, kTypeValue), "val"},
+ });
+ AddMockFile(file2);
+
+ auto expected_results = mock::MakeMockFile({
+ {KeyStr("dummy", 0U, kTypeValue), "val2"},
+ });
+
+ SetLastSequence(22U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results, {});
+}
+
+TEST_F(CompactionJobTest, MultiSingleDelete) {
+ // Tests three scenarios involving multiple single delete/put pairs:
+ //
+ // A: Put Snapshot SDel Put SDel -> Put Snapshot SDel
+ // B: Snapshot Put SDel Put SDel Snapshot -> Snapshot SDel Snapshot
+ // C: SDel Put SDel Snapshot Put -> Snapshot Put
+ // D: (Put) SDel Snapshot Put SDel -> (Put) SDel Snapshot SDel
+ // E: Put SDel Snapshot Put SDel -> Snapshot SDel
+ // F: Put SDel Put Sdel Snapshot -> removed
+ // G: Snapshot SDel Put SDel Put -> Snapshot Put SDel
+ // H: (Put) Put SDel Put Sdel Snapshot -> Removed
+ // I: (Put) Snapshot Put SDel Put SDel -> SDel
+ // J: Put Put SDel Put SDel SDel Snapshot Put Put SDel SDel Put
+ // -> Snapshot Put
+ // K: SDel SDel Put SDel Put Put Snapshot SDel Put SDel SDel Put SDel
+ // -> Snapshot Put Snapshot SDel
+ // L: SDel Put Del Put SDel Snapshot Del Put Del SDel Put SDel
+ // -> Snapshot SDel
+ // M: (Put) SDel Put Del Put SDel Snapshot Put Del SDel Put SDel Del
+ // -> SDel Snapshot Del
+ NewDB();
+
+ auto file1 = mock::MakeMockFile({
+ {KeyStr("A", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("A", 13U, kTypeValue), "val5"},
+ {KeyStr("A", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("B", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("B", 13U, kTypeValue), "val2"},
+ {KeyStr("C", 14U, kTypeValue), "val3"},
+ {KeyStr("D", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("D", 11U, kTypeValue), "val4"},
+ {KeyStr("G", 15U, kTypeValue), "val"},
+ {KeyStr("G", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("G", 13U, kTypeValue), "val"},
+ {KeyStr("I", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 13U, kTypeValue), "val"},
+ {KeyStr("J", 15U, kTypeValue), "val"},
+ {KeyStr("J", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 13U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 12U, kTypeValue), "val"},
+ {KeyStr("J", 11U, kTypeValue), "val"},
+ {KeyStr("K", 16U, kTypeSingleDeletion), ""},
+ {KeyStr("K", 15U, kTypeValue), "val1"},
+ {KeyStr("K", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("K", 13U, kTypeSingleDeletion), ""},
+ {KeyStr("K", 12U, kTypeValue), "val2"},
+ {KeyStr("K", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("L", 16U, kTypeSingleDeletion), ""},
+ {KeyStr("L", 15U, kTypeValue), "val"},
+ {KeyStr("L", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("L", 13U, kTypeDeletion), ""},
+ {KeyStr("L", 12U, kTypeValue), "val"},
+ {KeyStr("L", 11U, kTypeDeletion), ""},
+ {KeyStr("M", 16U, kTypeDeletion), ""},
+ {KeyStr("M", 15U, kTypeSingleDeletion), ""},
+ {KeyStr("M", 14U, kTypeValue), "val"},
+ {KeyStr("M", 13U, kTypeSingleDeletion), ""},
+ {KeyStr("M", 12U, kTypeDeletion), ""},
+ {KeyStr("M", 11U, kTypeValue), "val"},
+ });
+ AddMockFile(file1);
+
+ auto file2 = mock::MakeMockFile({
+ {KeyStr("A", 10U, kTypeValue), "val"},
+ {KeyStr("B", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("B", 11U, kTypeValue), "val2"},
+ {KeyStr("C", 10U, kTypeSingleDeletion), ""},
+ {KeyStr("C", 9U, kTypeValue), "val6"},
+ {KeyStr("C", 8U, kTypeSingleDeletion), ""},
+ {KeyStr("D", 10U, kTypeSingleDeletion), ""},
+ {KeyStr("E", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("E", 11U, kTypeValue), "val"},
+ {KeyStr("E", 5U, kTypeSingleDeletion), ""},
+ {KeyStr("E", 4U, kTypeValue), "val"},
+ {KeyStr("F", 6U, kTypeSingleDeletion), ""},
+ {KeyStr("F", 5U, kTypeValue), "val"},
+ {KeyStr("F", 4U, kTypeSingleDeletion), ""},
+ {KeyStr("F", 3U, kTypeValue), "val"},
+ {KeyStr("G", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("H", 6U, kTypeSingleDeletion), ""},
+ {KeyStr("H", 5U, kTypeValue), "val"},
+ {KeyStr("H", 4U, kTypeSingleDeletion), ""},
+ {KeyStr("H", 3U, kTypeValue), "val"},
+ {KeyStr("I", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 11U, kTypeValue), "val"},
+ {KeyStr("J", 6U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 5U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 4U, kTypeValue), "val"},
+ {KeyStr("J", 3U, kTypeSingleDeletion), ""},
+ {KeyStr("J", 2U, kTypeValue), "val"},
+ {KeyStr("K", 8U, kTypeValue), "val3"},
+ {KeyStr("K", 7U, kTypeValue), "val4"},
+ {KeyStr("K", 6U, kTypeSingleDeletion), ""},
+ {KeyStr("K", 5U, kTypeValue), "val5"},
+ {KeyStr("K", 2U, kTypeSingleDeletion), ""},
+ {KeyStr("K", 1U, kTypeSingleDeletion), ""},
+ {KeyStr("L", 5U, kTypeSingleDeletion), ""},
+ {KeyStr("L", 4U, kTypeValue), "val"},
+ {KeyStr("L", 3U, kTypeDeletion), ""},
+ {KeyStr("L", 2U, kTypeValue), "val"},
+ {KeyStr("L", 1U, kTypeSingleDeletion), ""},
+ {KeyStr("M", 10U, kTypeSingleDeletion), ""},
+ {KeyStr("M", 7U, kTypeValue), "val"},
+ {KeyStr("M", 5U, kTypeDeletion), ""},
+ {KeyStr("M", 4U, kTypeValue), "val"},
+ {KeyStr("M", 3U, kTypeSingleDeletion), ""},
+ });
+ AddMockFile(file2);
+
+ auto file3 = mock::MakeMockFile({
+ {KeyStr("D", 1U, kTypeValue), "val"},
+ {KeyStr("H", 1U, kTypeValue), "val"},
+ {KeyStr("I", 2U, kTypeValue), "val"},
+ });
+ AddMockFile(file3, 2);
+
+ auto file4 = mock::MakeMockFile({
+ {KeyStr("M", 1U, kTypeValue), "val"},
+ });
+ AddMockFile(file4, 2);
+
+ auto expected_results =
+ mock::MakeMockFile({{KeyStr("A", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("A", 13U, kTypeValue), ""},
+ {KeyStr("A", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("A", 10U, kTypeValue), "val"},
+ {KeyStr("B", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("B", 13U, kTypeValue), ""},
+ {KeyStr("C", 14U, kTypeValue), "val3"},
+ {KeyStr("D", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("D", 11U, kTypeValue), ""},
+ {KeyStr("D", 10U, kTypeSingleDeletion), ""},
+ {KeyStr("E", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("E", 11U, kTypeValue), ""},
+ {KeyStr("G", 15U, kTypeValue), "val"},
+ {KeyStr("G", 12U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 14U, kTypeSingleDeletion), ""},
+ {KeyStr("I", 13U, kTypeValue), ""},
+ {KeyStr("J", 15U, kTypeValue), "val"},
+ {KeyStr("K", 16U, kTypeSingleDeletion), ""},
+ {KeyStr("K", 15U, kTypeValue), ""},
+ {KeyStr("K", 11U, kTypeSingleDeletion), ""},
+ {KeyStr("K", 8U, kTypeValue), "val3"},
+ {KeyStr("L", 16U, kTypeSingleDeletion), ""},
+ {KeyStr("L", 15U, kTypeValue), ""},
+ {KeyStr("M", 16U, kTypeDeletion), ""},
+ {KeyStr("M", 3U, kTypeSingleDeletion), ""}});
+
+ SetLastSequence(22U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results, {10U}, 10U);
+}
+
+// This test documents the behavior where a corrupt key follows a deletion or a
+// single deletion and the (single) deletion gets removed while the corrupt key
+// gets written out. TODO(noetzli): We probably want a better way to treat
+// corrupt keys.
+TEST_F(CompactionJobTest, CorruptionAfterDeletion) {
+ NewDB();
+
+ auto file1 =
+ mock::MakeMockFile({{test::KeyStr("A", 6U, kTypeValue), "val3"},
+ {test::KeyStr("a", 5U, kTypeDeletion), ""},
+ {test::KeyStr("a", 4U, kTypeValue, true), "val"}});
+ AddMockFile(file1);
+
+ auto file2 =
+ mock::MakeMockFile({{test::KeyStr("b", 3U, kTypeSingleDeletion), ""},
+ {test::KeyStr("b", 2U, kTypeValue, true), "val"},
+ {test::KeyStr("c", 1U, kTypeValue), "val2"}});
+ AddMockFile(file2);
+
+ auto expected_results =
+ mock::MakeMockFile({{test::KeyStr("A", 0U, kTypeValue), "val3"},
+ {test::KeyStr("a", 0U, kTypeValue, true), "val"},
+ {test::KeyStr("b", 0U, kTypeValue, true), "val"},
+ {test::KeyStr("c", 0U, kTypeValue), "val2"}});
+
+ SetLastSequence(6U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results);
+}
+
+TEST_F(CompactionJobTest, OldestBlobFileNumber) {
+ NewDB();
+
+ // Note: blob1 is inlined TTL, so it will not be considered for the purposes
+ // of identifying the oldest referenced blob file. Similarly, blob6 will be
+ // ignored because it has TTL and hence refers to a TTL blob file.
+ const stl_wrappers::KVMap::value_type blob1(
+ KeyStr("a", 1U, kTypeBlobIndex), BlobStrInlinedTTL("foo", 1234567890ULL));
+ const stl_wrappers::KVMap::value_type blob2(KeyStr("b", 2U, kTypeBlobIndex),
+ BlobStr(59, 123456, 999));
+ const stl_wrappers::KVMap::value_type blob3(KeyStr("c", 3U, kTypeBlobIndex),
+ BlobStr(138, 1000, 1 << 8));
+ auto file1 = mock::MakeMockFile({blob1, blob2, blob3});
+ AddMockFile(file1);
+
+ const stl_wrappers::KVMap::value_type blob4(KeyStr("d", 4U, kTypeBlobIndex),
+ BlobStr(199, 3 << 10, 1 << 20));
+ const stl_wrappers::KVMap::value_type blob5(KeyStr("e", 5U, kTypeBlobIndex),
+ BlobStr(19, 6789, 333));
+ const stl_wrappers::KVMap::value_type blob6(
+ KeyStr("f", 6U, kTypeBlobIndex),
+ BlobStrTTL(5, 2048, 1 << 7, 1234567890ULL));
+ auto file2 = mock::MakeMockFile({blob4, blob5, blob6});
+ AddMockFile(file2);
+
+ const stl_wrappers::KVMap::value_type expected_blob1(
+ KeyStr("a", 0U, kTypeBlobIndex), blob1.second);
+ const stl_wrappers::KVMap::value_type expected_blob2(
+ KeyStr("b", 0U, kTypeBlobIndex), blob2.second);
+ const stl_wrappers::KVMap::value_type expected_blob3(
+ KeyStr("c", 0U, kTypeBlobIndex), blob3.second);
+ const stl_wrappers::KVMap::value_type expected_blob4(
+ KeyStr("d", 0U, kTypeBlobIndex), blob4.second);
+ const stl_wrappers::KVMap::value_type expected_blob5(
+ KeyStr("e", 0U, kTypeBlobIndex), blob5.second);
+ const stl_wrappers::KVMap::value_type expected_blob6(
+ KeyStr("f", 0U, kTypeBlobIndex), blob6.second);
+ auto expected_results =
+ mock::MakeMockFile({expected_blob1, expected_blob2, expected_blob3,
+ expected_blob4, expected_blob5, expected_blob6});
+
+ SetLastSequence(6U);
+ auto files = cfd_->current()->storage_info()->LevelFiles(0);
+ RunCompaction({files}, expected_results, std::vector<SequenceNumber>(),
+ kMaxSequenceNumber, /* output_level */ 1, /* verify */ true,
+ /* expected_oldest_blob_file_number */ 19);
+}
+
+} // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+}
+
+#else
+#include <stdio.h>
+
+int main(int /*argc*/, char** /*argv*/) {
+ fprintf(stderr,
+ "SKIPPED as CompactionJobStats is not supported in ROCKSDB_LITE\n");
+ return 0;
+}
+
+#endif // ROCKSDB_LITE