From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:54:28 +0200 Subject: Adding upstream version 18.2.2. Signed-off-by: Daniel Baumann --- src/rocksdb/utilities/fault_injection_fs.h | 584 +++++++++++++++++++++++++++++ 1 file changed, 584 insertions(+) create mode 100644 src/rocksdb/utilities/fault_injection_fs.h (limited to 'src/rocksdb/utilities/fault_injection_fs.h') diff --git a/src/rocksdb/utilities/fault_injection_fs.h b/src/rocksdb/utilities/fault_injection_fs.h new file mode 100644 index 000000000..53c9ccb6f --- /dev/null +++ b/src/rocksdb/utilities/fault_injection_fs.h @@ -0,0 +1,584 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright 2014 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// This test uses a custom FileSystem to keep track of the state of a file +// system the last "Sync". The data being written is cached in a "buffer". +// Only when "Sync" is called, the data will be persistent. It can similate +// file data loss (or entire files) not protected by a "Sync". For any of the +// FileSystem related operations, by specify the "IOStatus Error", a specific +// error can be returned when file system is not activated. + +#pragma once + +#include +#include +#include +#include + +#include "file/filename.h" +#include "rocksdb/file_system.h" +#include "util/mutexlock.h" +#include "util/random.h" +#include "util/thread_local.h" + +namespace ROCKSDB_NAMESPACE { + +class TestFSWritableFile; +class FaultInjectionTestFS; + +struct FSFileState { + std::string filename_; + ssize_t pos_; + ssize_t pos_at_last_sync_; + ssize_t pos_at_last_flush_; + std::string buffer_; + + explicit FSFileState(const std::string& filename) + : filename_(filename), + pos_(-1), + pos_at_last_sync_(-1), + pos_at_last_flush_(-1) {} + + FSFileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} + + bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; } + + IOStatus DropUnsyncedData(); + + IOStatus DropRandomUnsyncedData(Random* rand); +}; + +// A wrapper around WritableFileWriter* file +// is written to or sync'ed. +class TestFSWritableFile : public FSWritableFile { + public: + explicit TestFSWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr&& f, + FaultInjectionTestFS* fs); + virtual ~TestFSWritableFile(); + virtual IOStatus Append(const Slice& data, const IOOptions&, + IODebugContext*) override; + virtual IOStatus Append(const Slice& data, const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override; + virtual IOStatus Truncate(uint64_t size, const IOOptions& options, + IODebugContext* dbg) override { + return target_->Truncate(size, options, dbg); + } + virtual IOStatus Close(const IOOptions& options, + IODebugContext* dbg) override; + virtual IOStatus Flush(const IOOptions&, IODebugContext*) override; + virtual IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + virtual IOStatus RangeSync(uint64_t /*offset*/, uint64_t /*nbytes*/, + const IOOptions& options, + IODebugContext* dbg) override; + virtual bool IsSyncThreadSafe() const override { return true; } + virtual IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + IODebugContext* dbg) override { + return target_->PositionedAppend(data, offset, options, dbg); + } + IOStatus PositionedAppend(const Slice& data, uint64_t offset, + const IOOptions& options, + const DataVerificationInfo& verification_info, + IODebugContext* dbg) override; + virtual size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + virtual bool use_direct_io() const override { + return target_->use_direct_io(); + }; + + private: + FSFileState state_; // Need protection by mutex_ + FileOptions file_opts_; + std::unique_ptr target_; + bool writable_file_opened_; + FaultInjectionTestFS* fs_; + port::Mutex mutex_; +}; + +// A wrapper around WritableFileWriter* file +// is written to or sync'ed. +class TestFSRandomRWFile : public FSRandomRWFile { + public: + explicit TestFSRandomRWFile(const std::string& fname, + std::unique_ptr&& f, + FaultInjectionTestFS* fs); + virtual ~TestFSRandomRWFile(); + IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; + IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; + IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + bool use_direct_io() const override { return target_->use_direct_io(); }; + + private: + std::unique_ptr target_; + bool file_opened_; + FaultInjectionTestFS* fs_; +}; + +class TestFSRandomAccessFile : public FSRandomAccessFile { + public: + explicit TestFSRandomAccessFile(const std::string& fname, + std::unique_ptr&& f, + FaultInjectionTestFS* fs); + ~TestFSRandomAccessFile() override {} + IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) const override; + IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, + const IOOptions& options, IODebugContext* dbg) override; + size_t GetRequiredBufferAlignment() const override { + return target_->GetRequiredBufferAlignment(); + } + bool use_direct_io() const override { return target_->use_direct_io(); } + + size_t GetUniqueId(char* id, size_t max_size) const override; + + private: + std::unique_ptr target_; + FaultInjectionTestFS* fs_; +}; + +class TestFSSequentialFile : public FSSequentialFileOwnerWrapper { + public: + explicit TestFSSequentialFile(std::unique_ptr&& f, + FaultInjectionTestFS* fs) + : FSSequentialFileOwnerWrapper(std::move(f)), fs_(fs) {} + IOStatus Read(size_t n, const IOOptions& options, Slice* result, + char* scratch, IODebugContext* dbg) override; + IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options, + Slice* result, char* scratch, + IODebugContext* dbg) override; + + private: + FaultInjectionTestFS* fs_; +}; + +class TestFSDirectory : public FSDirectory { + public: + explicit TestFSDirectory(FaultInjectionTestFS* fs, std::string dirname, + FSDirectory* dir) + : fs_(fs), dirname_(dirname), dir_(dir) {} + ~TestFSDirectory() {} + + virtual IOStatus Fsync(const IOOptions& options, + IODebugContext* dbg) override; + + virtual IOStatus Close(const IOOptions& options, + IODebugContext* dbg) override; + + virtual IOStatus FsyncWithDirOptions( + const IOOptions& options, IODebugContext* dbg, + const DirFsyncOptions& dir_fsync_options) override; + + private: + FaultInjectionTestFS* fs_; + std::string dirname_; + std::unique_ptr dir_; +}; + +class FaultInjectionTestFS : public FileSystemWrapper { + public: + explicit FaultInjectionTestFS(const std::shared_ptr& base) + : FileSystemWrapper(base), + filesystem_active_(true), + filesystem_writable_(false), + thread_local_error_(new ThreadLocalPtr(DeleteThreadLocalErrorContext)), + enable_write_error_injection_(false), + enable_metadata_write_error_injection_(false), + write_error_rand_(0), + write_error_one_in_(0), + metadata_write_error_one_in_(0), + read_error_one_in_(0), + ingest_data_corruption_before_write_(false), + fail_get_file_unique_id_(false) {} + virtual ~FaultInjectionTestFS() { error_.PermitUncheckedError(); } + + static const char* kClassName() { return "FaultInjectionTestFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewDirectory(const std::string& name, const IOOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + + virtual IOStatus DeleteFile(const std::string& f, const IOOptions& options, + IODebugContext* dbg) override; + + virtual IOStatus RenameFile(const std::string& s, const std::string& t, + const IOOptions& options, + IODebugContext* dbg) override; + + virtual IOStatus LinkFile(const std::string& src, const std::string& target, + const IOOptions& options, + IODebugContext* dbg) override; + +// Undef to eliminate clash on Windows +#undef GetFreeSpace + virtual IOStatus GetFreeSpace(const std::string& path, + const IOOptions& options, uint64_t* disk_free, + IODebugContext* dbg) override { + IOStatus io_s; + if (!IsFilesystemActive() && + error_.subcode() == IOStatus::SubCode::kNoSpace) { + *disk_free = 0; + } else { + io_s = target()->GetFreeSpace(path, options, disk_free, dbg); + } + return io_s; + } + + void WritableFileClosed(const FSFileState& state); + + void WritableFileSynced(const FSFileState& state); + + void WritableFileAppended(const FSFileState& state); + + IOStatus DropUnsyncedFileData(); + + IOStatus DropRandomUnsyncedFileData(Random* rnd); + + IOStatus DeleteFilesCreatedAfterLastDirSync(const IOOptions& options, + IODebugContext* dbg); + + void ResetState(); + + void UntrackFile(const std::string& f); + + void SyncDir(const std::string& dirname) { + MutexLock l(&mutex_); + dir_to_new_files_since_last_sync_.erase(dirname); + } + + // Setting the filesystem to inactive is the test equivalent to simulating a + // system reset. Setting to inactive will freeze our saved filesystem state so + // that it will stop being recorded. It can then be reset back to the state at + // the time of the reset. + bool IsFilesystemActive() { + MutexLock l(&mutex_); + return filesystem_active_; + } + + // Setting filesystem_writable_ makes NewWritableFile. ReopenWritableFile, + // and NewRandomRWFile bypass FaultInjectionTestFS and go directly to the + // target FS + bool IsFilesystemDirectWritable() { + MutexLock l(&mutex_); + return filesystem_writable_; + } + bool ShouldUseDiretWritable(const std::string& file_name) { + MutexLock l(&mutex_); + if (filesystem_writable_) { + return true; + } + FileType file_type = kTempFile; + uint64_t file_number = 0; + if (!TryParseFileName(file_name, &file_number, &file_type)) { + return false; + } + return skip_direct_writable_types_.find(file_type) != + skip_direct_writable_types_.end(); + } + void SetFilesystemActiveNoLock( + bool active, IOStatus error = IOStatus::Corruption("Not active")) { + error.PermitUncheckedError(); + filesystem_active_ = active; + if (!active) { + error_ = error; + } + } + void SetFilesystemActive( + bool active, IOStatus error = IOStatus::Corruption("Not active")) { + MutexLock l(&mutex_); + error.PermitUncheckedError(); + SetFilesystemActiveNoLock(active, error); + } + void SetFilesystemDirectWritable(bool writable) { + MutexLock l(&mutex_); + filesystem_writable_ = writable; + } + void AssertNoOpenFile() { assert(open_managed_files_.empty()); } + + IOStatus GetError() { return error_; } + + void SetFileSystemIOError(IOStatus io_error) { + MutexLock l(&mutex_); + io_error.PermitUncheckedError(); + error_ = io_error; + } + + // To simulate the data corruption before data is written in FS + void IngestDataCorruptionBeforeWrite() { + MutexLock l(&mutex_); + ingest_data_corruption_before_write_ = true; + } + + void NoDataCorruptionBeforeWrite() { + MutexLock l(&mutex_); + ingest_data_corruption_before_write_ = false; + } + + bool ShouldDataCorruptionBeforeWrite() { + MutexLock l(&mutex_); + return ingest_data_corruption_before_write_; + } + + void SetChecksumHandoffFuncType(const ChecksumType& func_type) { + MutexLock l(&mutex_); + checksum_handoff_func_tpye_ = func_type; + } + + const ChecksumType& GetChecksumHandoffFuncType() { + MutexLock l(&mutex_); + return checksum_handoff_func_tpye_; + } + + void SetFailGetUniqueId(bool flag) { + MutexLock l(&mutex_); + fail_get_file_unique_id_ = flag; + } + + bool ShouldFailGetUniqueId() { + MutexLock l(&mutex_); + return fail_get_file_unique_id_; + } + + // Specify what the operation, so we can inject the right type of error + enum ErrorOperation : char { + kRead = 0, + kMultiReadSingleReq = 1, + kMultiRead = 2, + kOpen, + }; + + // Set thread-local parameters for error injection. The first argument, + // seed is the seed for the random number generator, and one_in determines + // the probability of injecting error (i.e an error is injected with + // 1/one_in probability) + void SetThreadLocalReadErrorContext(uint32_t seed, int one_in) { + struct ErrorContext* ctx = + static_cast(thread_local_error_->Get()); + if (ctx == nullptr) { + ctx = new ErrorContext(seed); + thread_local_error_->Reset(ctx); + } + ctx->one_in = one_in; + ctx->count = 0; + } + + static void DeleteThreadLocalErrorContext(void* p) { + ErrorContext* ctx = static_cast(p); + delete ctx; + } + + // This is to set the parameters for the write error injection. + // seed is the seed for the random number generator, and one_in determines + // the probability of injecting error (i.e an error is injected with + // 1/one_in probability). For write error, we can specify the error we + // want to inject. Types decides the file types we want to inject the + // error (e.g., Wal files, SST files), which is empty by default. + void SetRandomWriteError(uint32_t seed, int one_in, IOStatus error, + bool inject_for_all_file_types, + const std::vector& types) { + MutexLock l(&mutex_); + Random tmp_rand(seed); + error.PermitUncheckedError(); + error_ = error; + write_error_rand_ = tmp_rand; + write_error_one_in_ = one_in; + inject_for_all_file_types_ = inject_for_all_file_types; + write_error_allowed_types_ = types; + } + + void SetSkipDirectWritableTypes(const std::set& types) { + MutexLock l(&mutex_); + skip_direct_writable_types_ = types; + } + + void SetRandomMetadataWriteError(int one_in) { + MutexLock l(&mutex_); + metadata_write_error_one_in_ = one_in; + } + // If the value is not 0, it is enabled. Otherwise, it is disabled. + void SetRandomReadError(int one_in) { read_error_one_in_ = one_in; } + + bool ShouldInjectRandomReadError() { + return read_error_one_in() && + Random::GetTLSInstance()->OneIn(read_error_one_in()); + } + + // Inject an write error with randomlized parameter and the predefined + // error type. Only the allowed file types will inject the write error + IOStatus InjectWriteError(const std::string& file_name); + + // Ingest error to metadata operations. + IOStatus InjectMetadataWriteError(); + + // Inject an error. For a READ operation, a status of IOError(), a + // corruption in the contents of scratch, or truncation of slice + // are the types of error with equal probability. For OPEN, + // its always an IOError. + // fault_injected returns whether a fault is injected. It is needed + // because some fault is inected with IOStatus to be OK. + IOStatus InjectThreadSpecificReadError(ErrorOperation op, Slice* slice, + bool direct_io, char* scratch, + bool need_count_increase, + bool* fault_injected); + + // Get the count of how many times we injected since the previous call + int GetAndResetErrorCount() { + ErrorContext* ctx = static_cast(thread_local_error_->Get()); + int count = 0; + if (ctx != nullptr) { + count = ctx->count; + ctx->count = 0; + } + return count; + } + + void EnableErrorInjection() { + ErrorContext* ctx = static_cast(thread_local_error_->Get()); + if (ctx) { + ctx->enable_error_injection = true; + } + } + + void EnableWriteErrorInjection() { + MutexLock l(&mutex_); + enable_write_error_injection_ = true; + } + void EnableMetadataWriteErrorInjection() { + MutexLock l(&mutex_); + enable_metadata_write_error_injection_ = true; + } + + void DisableWriteErrorInjection() { + MutexLock l(&mutex_); + enable_write_error_injection_ = false; + } + + void DisableErrorInjection() { + ErrorContext* ctx = static_cast(thread_local_error_->Get()); + if (ctx) { + ctx->enable_error_injection = false; + } + } + + void DisableMetadataWriteErrorInjection() { + MutexLock l(&mutex_); + enable_metadata_write_error_injection_ = false; + } + + int read_error_one_in() const { return read_error_one_in_.load(); } + + int write_error_one_in() const { return write_error_one_in_; } + + // We capture a backtrace every time a fault is injected, for debugging + // purposes. This call prints the backtrace to stderr and frees the + // saved callstack + void PrintFaultBacktrace(); + + private: + port::Mutex mutex_; + std::map db_file_state_; + std::set open_managed_files_; + // directory -> (file name -> file contents to recover) + // When data is recovered from unsyned parent directory, the files with + // empty file contents to recover is deleted. Those with non-empty ones + // will be recovered to content accordingly. + std::unordered_map> + dir_to_new_files_since_last_sync_; + bool filesystem_active_; // Record flushes, syncs, writes + bool filesystem_writable_; // Bypass FaultInjectionTestFS and go directly + // to underlying FS for writable files + IOStatus error_; + + enum ErrorType : int { + kErrorTypeStatus = 0, + kErrorTypeCorruption, + kErrorTypeTruncated, + kErrorTypeMax + }; + + struct ErrorContext { + Random rand; + int one_in; + int count; + bool enable_error_injection; + void* callstack; + std::string message; + int frames; + ErrorType type; + + explicit ErrorContext(uint32_t seed) + : rand(seed), + enable_error_injection(false), + callstack(nullptr), + frames(0) {} + ~ErrorContext() { + if (callstack) { + free(callstack); + } + } + }; + + std::unique_ptr thread_local_error_; + bool enable_write_error_injection_; + bool enable_metadata_write_error_injection_; + Random write_error_rand_; + int write_error_one_in_; + int metadata_write_error_one_in_; + std::atomic read_error_one_in_; + bool inject_for_all_file_types_; + std::vector write_error_allowed_types_; + // File types where direct writable is skipped. + std::set skip_direct_writable_types_; + bool ingest_data_corruption_before_write_; + ChecksumType checksum_handoff_func_tpye_; + bool fail_get_file_unique_id_; + + // Extract number of type from file name. Return false if failing to fine + // them. + bool TryParseFileName(const std::string& file_name, uint64_t* number, + FileType* type); +}; + +} // namespace ROCKSDB_NAMESPACE -- cgit v1.2.3