diff options
Diffstat (limited to 'src/test/crimson')
35 files changed, 10604 insertions, 0 deletions
diff --git a/src/test/crimson/CMakeLists.txt b/src/test/crimson/CMakeLists.txt new file mode 100644 index 000000000..5a0de43d2 --- /dev/null +++ b/src/test/crimson/CMakeLists.txt @@ -0,0 +1,90 @@ +# the crimson's backfill doesn't need nor use seastar +add_executable(unittest-crimson-backfill + test_backfill.cc + ${PROJECT_SOURCE_DIR}/src/auth/Crypto.cc + ${PROJECT_SOURCE_DIR}/src/crimson/osd/backfill_state.cc + ${PROJECT_SOURCE_DIR}/src/osd/recovery_types.cc) +add_ceph_unittest(unittest-crimson-backfill + --memory 256M --smp 1) +target_link_libraries(unittest-crimson-backfill crimson GTest::Main) + +add_executable(unittest-seastar-buffer + test_buffer.cc) +add_ceph_unittest(unittest-seastar-buffer + --memory 256M --smp 1) +target_link_libraries(unittest-seastar-buffer crimson) + +add_executable(unittest-seastar-denc + test_denc.cc) +add_ceph_unittest(unittest-seastar-denc --memory 256M --smp 1) +target_link_libraries(unittest-seastar-denc crimson GTest::Main) + +add_executable(unittest-seastar-socket test_socket.cc) +add_ceph_unittest(unittest-seastar-socket + --memory 256M --smp 2) +target_link_libraries(unittest-seastar-socket crimson) + +add_executable(unittest-seastar-messenger test_messenger.cc) +add_ceph_unittest(unittest-seastar-messenger + --memory 256M --smp 1) +target_link_libraries(unittest-seastar-messenger crimson) + +add_executable(test-seastar-messenger-peer test_messenger_peer.cc) +target_link_libraries(test-seastar-messenger-peer ceph-common global ${ALLOC_LIBS}) + +add_executable(test-seastar-echo + test_alien_echo.cc) +target_link_libraries(test-seastar-echo crimson) + +add_executable(test-async-echo + test_async_echo.cc) +target_link_libraries(test-async-echo ceph-common global) + +add_executable(unittest-seastar-alienstore-thread-pool + test_alienstore_thread_pool.cc) +add_ceph_unittest(unittest-seastar-alienstore-thread-pool + --memory 256M --smp 1) +target_link_libraries(unittest-seastar-alienstore-thread-pool + crimson-alienstore + crimson) + +add_executable(unittest-seastar-config + test_config.cc) +add_ceph_unittest(unittest-seastar-config + --memory 256M --smp 4) +target_link_libraries(unittest-seastar-config crimson) + +add_executable(unittest-seastar-monc + test_monc.cc) +target_link_libraries(unittest-seastar-monc crimson) + +add_executable(unittest-seastar-perfcounters + test_perfcounters.cc) +add_ceph_unittest(unittest-seastar-perfcounters + --memory 256M --smp 1) +target_link_libraries(unittest-seastar-perfcounters crimson) + +add_executable(unittest-seastar-lru + test_lru.cc) +add_ceph_unittest(unittest-seastar-lru + --memory 256M --smp 1) +target_link_libraries(unittest-seastar-lru crimson GTest::Main) + +add_executable(unittest-fixed-kv-node-layout + test_fixed_kv_node_layout.cc) +add_ceph_unittest(unittest-fixed-kv-node-layout) + +add_subdirectory(seastore) + +add_library(crimson-gtest STATIC + gtest_seastar.cc) +target_link_libraries(crimson-gtest crimson-common GTest::GTest) +add_library(crimson::gtest ALIAS crimson-gtest) + +add_executable(unittest-seastar-errorator + test_errorator.cc) +target_link_libraries( + unittest-seastar-errorator + crimson::gtest) +add_ceph_unittest(unittest-seastar-errorator + --memory 256M --smp 1) diff --git a/src/test/crimson/cbt/radosbench_4K_read.yaml b/src/test/crimson/cbt/radosbench_4K_read.yaml new file mode 100644 index 000000000..219ce643a --- /dev/null +++ b/src/test/crimson/cbt/radosbench_4K_read.yaml @@ -0,0 +1,36 @@ +meta: +- desc: | + Run radosbench benchmark using cbt. + 4K read workload. + +tasks: +- cbt: + benchmarks: + radosbench: + concurrent_ops: 16 + concurrent_procs: 2 + op_size: [4096] + pool_profile: 'replicated' + read_time: 30 + read_only: true + readmode: 'rand' + prefill_time: 3 + acceptable: + bandwidth: '(or (greater) (near 0.05))' + iops_avg: '(or (greater) (near 0.05))' + iops_stddev: '(or (less) (near 2.00))' + latency_avg: '(or (less) (near 0.05))' + cpu_cycles_per_op: '(or (less) (near 0.05))' + monitoring_profiles: + perf: + nodes: + - osds + args: 'stat -p {pid} -o {perf_dir}/perf_stat.{pid}' + cluster: + osds_per_node: 3 + iterations: 1 + pool_profiles: + replicated: + pg_size: 128 + pgp_size: 128 + replication: 'replicated' diff --git a/src/test/crimson/cbt/radosbench_4K_write.yaml b/src/test/crimson/cbt/radosbench_4K_write.yaml new file mode 100644 index 000000000..526982b10 --- /dev/null +++ b/src/test/crimson/cbt/radosbench_4K_write.yaml @@ -0,0 +1,34 @@ +meta: +- desc: | + Run radosbench benchmark using cbt. + 4K write workload. + +tasks: +- cbt: + benchmarks: + radosbench: + concurrent_ops: 16 + concurrent_procs: 2 + op_size: [4096] + pool_profile: 'replicated' + write_time: 3 + write_only: true + acceptable: + bandwidth: '(or (greater) (near 0.05))' + iops_avg: '(or (greater) (near 0.05))' + iops_stddev: '(or (less) (near 2.00))' + latency_avg: '(or (less) (near 0.05))' + cpu_cycles_per_op: '(or (less) (near 0.05))' + monitoring_profiles: + perf: + nodes: + - osds + args: 'stat -p {pid} -o {perf_dir}/perf_stat.{pid}' + cluster: + osds_per_node: 3 + iterations: 1 + pool_profiles: + replicated: + pg_size: 128 + pgp_size: 128 + replication: 'replicated' diff --git a/src/test/crimson/cbt/t2c.py b/src/test/crimson/cbt/t2c.py new file mode 100755 index 000000000..0d4ee49e5 --- /dev/null +++ b/src/test/crimson/cbt/t2c.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +from __future__ import print_function +import argparse +import os +import os.path +import socket +import sys +import yaml + + +class Translator(object): + def __init__(self, build_dir): + self.build_dir = build_dir + + def translate(self, config): + cluster = config.get('cluster', {}) + benchmarks = config.get('benchmarks', []) + monitoring_profiles = config.get('monitoring_profiles', {}) + return dict(cluster=self._create_cluster_config(cluster), + benchmarks=benchmarks, + monitoring_profiles=monitoring_profiles) + + def _create_cluster_config(self, cluster): + # prepare the "cluster" section consumed by CBT + localhost = socket.getfqdn() + num_osds = cluster.get('osds_per_node', 3) + items_to_copy = ['iterations', 'pool_profiles'] + conf = dict((k, cluster[k]) for k in items_to_copy if k in cluster) + conf.update(dict( + head=localhost, + osds=[localhost], + osds_per_node=num_osds, + mons=[localhost], + clients=[localhost], + rebuild_every_test=False, + conf_file=os.path.join(self.build_dir, 'ceph.conf'), + ceph_cmd=os.path.join(self.build_dir, 'bin', 'ceph'), + rados_cmd=os.path.join(self.build_dir, 'bin', 'rados'), + pid_dir=os.path.join(self.build_dir, 'out') + )) + return conf + +def get_cbt_tasks(path): + with open(path) as input: + teuthology_config = yaml.load(input) + for task in teuthology_config['tasks']: + for name, conf in task.items(): + if name == 'cbt': + yield conf + +def main(): + parser = argparse.ArgumentParser(description='translate teuthology yaml to CBT yaml') + parser.add_argument('--build-dir', + default=os.getcwd(), + required=False, + help='Directory where CMakeCache.txt is located') + parser.add_argument('--input', + required=True, + help='The path to the input YAML file') + parser.add_argument('--output', + required=True, + help='The path to the output YAML file') + options = parser.parse_args(sys.argv[1:]) + cbt_tasks = [task for task in get_cbt_tasks(options.input)] + if not cbt_tasks: + print('cbt not found in "tasks" section', file=sys.stderr) + return sys.exit(1) + elif len(cbt_tasks) > 1: + print('more than one cbt task found in "tasks" section', file=sys.stderr) + return sys.exit(1) + translator = Translator(options.build_dir) + cbt_config = translator.translate(cbt_tasks[0]) + with open(options.output, 'w') as output: + yaml.dump(cbt_config, output) + +if __name__ == '__main__': + main() diff --git a/src/test/crimson/gtest_seastar.cc b/src/test/crimson/gtest_seastar.cc new file mode 100644 index 000000000..9d43fc6be --- /dev/null +++ b/src/test/crimson/gtest_seastar.cc @@ -0,0 +1,23 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/ceph_assert.h" +#include "gtest_seastar.h" + +SeastarRunner seastar_test_suite_t::seastar_env; + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + + seastar_test_suite_t::seastar_env.init(argc, argv); + + seastar::global_logger_registry().set_all_loggers_level( + seastar::log_level::debug + ); + + int ret = RUN_ALL_TESTS(); + + seastar_test_suite_t::seastar_env.stop(); + return ret; +} diff --git a/src/test/crimson/gtest_seastar.h b/src/test/crimson/gtest_seastar.h new file mode 100644 index 000000000..20709a3ee --- /dev/null +++ b/src/test/crimson/gtest_seastar.h @@ -0,0 +1,35 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include "gtest/gtest.h" + +#include "seastar_runner.h" + +struct seastar_test_suite_t : public ::testing::Test { + static SeastarRunner seastar_env; + + template <typename Func> + void run(Func &&func) { + return seastar_env.run(std::forward<Func>(func)); + } + + template <typename Func> + void run_async(Func &&func) { + run( + [func=std::forward<Func>(func)]() mutable { + return seastar::async(std::forward<Func>(func)); + }); + } + + virtual seastar::future<> set_up_fut() { return seastar::now(); } + void SetUp() final { + return run([this] { return set_up_fut(); }); + } + + virtual seastar::future<> tear_down_fut() { return seastar::now(); } + void TearDown() final { + return run([this] { return tear_down_fut(); }); + } +}; diff --git a/src/test/crimson/seastar_runner.h b/src/test/crimson/seastar_runner.h new file mode 100644 index 000000000..5a430554e --- /dev/null +++ b/src/test/crimson/seastar_runner.h @@ -0,0 +1,71 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include <stdio.h> +#include <signal.h> +#include <thread> + +#include <seastar/core/app-template.hh> +#include <seastar/core/future-util.hh> +#include <seastar/core/reactor.hh> +#include <seastar/core/alien.hh> +#include <seastar/core/thread.hh> + +struct SeastarRunner { + seastar::app_template app; + seastar::file_desc begin_fd; + std::unique_ptr<seastar::readable_eventfd> on_end; + + std::thread thread; + + SeastarRunner() : + begin_fd{seastar::file_desc::eventfd(0, 0)} {} + + ~SeastarRunner() {} + + void init(int argc, char **argv) + { + thread = std::thread([argc, argv, this] { reactor(argc, argv); }); + eventfd_t result = 0; + if (int r = ::eventfd_read(begin_fd.get(), &result); r < 0) { + std::cerr << "unable to eventfd_read():" << errno << std::endl; + throw std::runtime_error("Cannot start seastar"); + } + } + + void stop() + { + run([this] { + on_end->write_side().signal(1); + return seastar::now(); + }); + thread.join(); + } + + void reactor(int argc, char **argv) + { + app.run(argc, argv, [this] { + on_end.reset(new seastar::readable_eventfd); + return seastar::now().then([this] { + ::eventfd_write(begin_fd.get(), 1); + return seastar::now(); + }).then([this] { + return on_end->wait().then([](size_t){}); + }).handle_exception([](auto ep) { + std::cerr << "Error: " << ep << std::endl; + }).finally([this] { + on_end.reset(); + }); + }); + } + + template <typename Func> + void run(Func &&func) { + auto fut = seastar::alien::submit_to(0, std::forward<Func>(func)); + fut.get(); + } +}; + + diff --git a/src/test/crimson/seastore/CMakeLists.txt b/src/test/crimson/seastore/CMakeLists.txt new file mode 100644 index 000000000..6c21ac7c5 --- /dev/null +++ b/src/test/crimson/seastore/CMakeLists.txt @@ -0,0 +1,51 @@ +add_executable(unittest-transaction-manager + test_block.cc + test_transaction_manager.cc + ../gtest_seastar.cc) +add_ceph_unittest(unittest-transaction-manager + --memory 256M --smp 1) +target_link_libraries( + unittest-transaction-manager + ${CMAKE_DL_LIBS} + crimson-seastore) + +add_executable(unittest-btree-lba-manager + test_btree_lba_manager.cc + ../gtest_seastar.cc) +add_ceph_unittest(unittest-btree-lba-manager + --memory 256M --smp 1) +target_link_libraries( + unittest-btree-lba-manager + ${CMAKE_DL_LIBS} + crimson-seastore) + +add_executable(unittest-seastore-journal + test_seastore_journal.cc) +add_ceph_test(unittest-seastore-journal + unittest-seastore-journal --memory 256M --smp 1) +target_link_libraries( + unittest-seastore-journal + crimson::gtest + crimson-seastore) + +add_executable(unittest-seastore-cache + test_block.cc + test_seastore_cache.cc) +add_ceph_test(unittest-seastore-cache + unittest-seastore-cache --memory 256M --smp 1) +target_link_libraries( + unittest-seastore-cache + crimson::gtest + crimson-seastore) + +add_executable(unittest-extmap-manager + test_extmap_manager.cc + ../gtest_seastar.cc) +add_ceph_unittest(unittest-extmap-manager + --memory 256M --smp 1) +target_link_libraries( + unittest-extmap-manager + ${CMAKE_DL_LIBS} + crimson-seastore) + +add_subdirectory(onode_tree) diff --git a/src/test/crimson/seastore/onode_tree/CMakeLists.txt b/src/test/crimson/seastore/onode_tree/CMakeLists.txt new file mode 100644 index 000000000..0886d2fb6 --- /dev/null +++ b/src/test/crimson/seastore/onode_tree/CMakeLists.txt @@ -0,0 +1,15 @@ +add_executable(test-seastore-onode-tree-node + test_node.cc) +add_ceph_unittest(test-seastore-onode-tree-node + --memory 256M --smp 1) +target_link_libraries(test-seastore-onode-tree-node + crimson-seastore + GTest::Main) + +add_executable(unittest-staged-fltree + test_staged_fltree.cc + ../../gtest_seastar.cc) +add_ceph_unittest(unittest-staged-fltree + --memory 256M --smp 1) +target_link_libraries(unittest-staged-fltree + crimson-seastore) diff --git a/src/test/crimson/seastore/onode_tree/test_node.cc b/src/test/crimson/seastore/onode_tree/test_node.cc new file mode 100644 index 000000000..178f78365 --- /dev/null +++ b/src/test/crimson/seastore/onode_tree/test_node.cc @@ -0,0 +1,207 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include <gtest/gtest.h> + +#include "crimson/os/seastore/onode_manager/simple-fltree/onode_node.h" + +using crimson::os::seastore::Onode; +using crimson::os::seastore::OnodeRef; + +TEST(OnodeNode, denc) +{ + Onode onode{"hello"}; + bufferlist bl; + ceph::encode(onode, bl); + bl.rebuild(); + auto flattened = reinterpret_cast<const onode_t*>(bl.c_str()); + auto actual_onode = flattened->decode(); + ASSERT_EQ(*actual_onode, onode); +} + +TEST(OnodeNode, lookup) +{ + static constexpr size_t BLOCK_SIZE = 512; + char buf[BLOCK_SIZE]; + using leaf_node_0 = node_t<BLOCK_SIZE, 0, ntype_t::leaf>; + auto leaf = new (buf) leaf_node_0; + ghobject_t oid{hobject_t{object_t{"saturn"}, "", 0, 0, 0, "solar"}}; + { + auto [slot, found] = leaf->lower_bound(oid); + ASSERT_FALSE(found); + ASSERT_EQ(0, slot); + } + Onode onode{"hello"}; + bufferlist bl; + ceph::encode(onode, bl); + bl.rebuild(); + auto flattened = reinterpret_cast<const onode_t*>(bl.c_str()); + leaf->insert_at(0, oid, *flattened); + { + auto [slot, found] = leaf->lower_bound(oid); + ASSERT_TRUE(found); + ASSERT_EQ(0, slot); + const auto& [key1, key2] = leaf->key_at(slot); + auto& item = leaf->item_at(key1); + auto actual_onode = item.decode(); + ASSERT_EQ(*actual_onode, onode); + } +} + +TEST(OnodeNode, grab_from_right) +{ + static constexpr size_t BLOCK_SIZE = 512; + char buf1[BLOCK_SIZE]; + char buf2[BLOCK_SIZE]; + using leaf_node_0 = node_t<BLOCK_SIZE, 0, ntype_t::leaf>; + auto leaf1 = new (buf1) leaf_node_0; + auto leaf2 = new (buf2) leaf_node_0; + auto& dummy_parent = *leaf1; + + ghobject_t oid1{hobject_t{object_t{"earth"}, "", 0, 0, 0, "solar"}}; + ghobject_t oid2{hobject_t{object_t{"jupiter"}, "", 0, 0, 0, "solar"}}; + ghobject_t oid3{hobject_t{object_t{"saturn"}, "", 0, 0, 0, "solar"}}; + Onode onode{"hello"}; + bufferlist bl; + ceph::encode(onode, bl); + bl.rebuild(); + auto flattened = reinterpret_cast<const onode_t*>(bl.c_str()); + // so they are ordered as they should + leaf1->insert_at(0, oid1, *flattened); + ASSERT_EQ(1, leaf1->count); + { + auto [slot, found] = leaf1->lower_bound(oid1); + ASSERT_TRUE(found); + ASSERT_EQ(0, slot); + } + { + leaf2->insert_at(0, oid2, *flattened); + auto [slot, found] = leaf2->lower_bound(oid2); + ASSERT_TRUE(found); + ASSERT_EQ(0, slot); + } + { + leaf2->insert_at(1, oid3, *flattened); + auto [slot, found] = leaf2->lower_bound(oid3); + ASSERT_TRUE(found); + ASSERT_EQ(1, slot); + } + ASSERT_EQ(2, leaf2->count); + + // normally we let left merge right, so we just need to remove an + // entry in parent, let's keep this convention here + auto mover = make_mover(dummy_parent, *leaf2, *leaf1, 0); + // just grab a single item from right + mover.move_from(0, 1, 1); + auto to_delta = mover.to_delta(); + ASSERT_EQ(to_delta.op_t::insert_back, to_delta.op); + leaf1->insert_back(std::move(to_delta.keys), std::move(to_delta.cells)); + + ASSERT_EQ(2, leaf1->count); + { + auto [slot, found] = leaf1->lower_bound(oid2); + ASSERT_TRUE(found); + ASSERT_EQ(1, slot); + } + + auto from_delta = mover.from_delta(); + ASSERT_EQ(from_delta.op_t::shift_left, from_delta.op); + leaf2->shift_left(from_delta.n, 0); + ASSERT_EQ(1, leaf2->count); +} + +TEST(OnodeNode, merge_right) +{ + static constexpr size_t BLOCK_SIZE = 512; + char buf1[BLOCK_SIZE]; + char buf2[BLOCK_SIZE]; + using leaf_node_0 = node_t<BLOCK_SIZE, 0, ntype_t::leaf>; + auto leaf1 = new (buf1) leaf_node_0; + auto leaf2 = new (buf2) leaf_node_0; + auto& dummy_parent = leaf1; + + ghobject_t oid1{hobject_t{object_t{"earth"}, "", 0, 0, 0, "solar"}}; + ghobject_t oid2{hobject_t{object_t{"jupiter"}, "", 0, 0, 0, "solar"}}; + ghobject_t oid3{hobject_t{object_t{"saturn"}, "", 0, 0, 0, "solar"}}; + Onode onode{"hello"}; + bufferlist bl; + ceph::encode(onode, bl); + bl.rebuild(); + auto flattened = reinterpret_cast<const onode_t*>(bl.c_str()); + // so they are ordered as they should + leaf1->insert_at(0, oid1, *flattened); + ASSERT_EQ(1, leaf1->count); + { + auto [slot, found] = leaf1->lower_bound(oid1); + ASSERT_TRUE(found); + ASSERT_EQ(0, slot); + } + { + leaf2->insert_at(0, oid2, *flattened); + auto [slot, found] = leaf2->lower_bound(oid2); + ASSERT_TRUE(found); + ASSERT_EQ(0, slot); + } + { + leaf2->insert_at(1, oid3, *flattened); + auto [slot, found] = leaf2->lower_bound(oid3); + ASSERT_TRUE(found); + ASSERT_EQ(1, slot); + } + ASSERT_EQ(2, leaf2->count); + + // normally we let left merge right, so we just need to remove an + // entry in parent, let's keep this convention here + auto mover = make_mover(dummy_parent, *leaf2, *leaf1, 0); + // just grab a single item from right + mover.move_from(0, 1, 2); + auto to_delta = mover.to_delta(); + ASSERT_EQ(to_delta.op_t::insert_back, to_delta.op); + leaf1->insert_back(std::move(to_delta.keys), std::move(to_delta.cells)); + + ASSERT_EQ(3, leaf1->count); + { + auto [slot, found] = leaf1->lower_bound(oid2); + ASSERT_TRUE(found); + ASSERT_EQ(1, slot); + } + { + auto [slot, found] = leaf1->lower_bound(oid3); + ASSERT_TRUE(found); + ASSERT_EQ(2, slot); + } + + // its onode tree's responsibility to retire the node + auto from_delta = mover.from_delta(); + ASSERT_EQ(from_delta.op_t::nop, from_delta.op); +} + +TEST(OnodeNode, remove_basic) +{ + static constexpr size_t BLOCK_SIZE = 512; + char buf[BLOCK_SIZE]; + using leaf_node_0 = node_t<BLOCK_SIZE, 0, ntype_t::leaf>; + auto leaf = new (buf) leaf_node_0; + ghobject_t oid{hobject_t{object_t{"saturn"}, "", 0, 0, 0, "solar"}}; + { + auto [slot, found] = leaf->lower_bound(oid); + ASSERT_FALSE(found); + ASSERT_EQ(0, slot); + } + Onode onode{"hello"}; + bufferlist bl; + ceph::encode(onode, bl); + bl.rebuild(); + auto flattened = reinterpret_cast<const onode_t*>(bl.c_str()); + leaf->insert_at(0, oid, *flattened); + { + auto [slot, found] = leaf->lower_bound(oid); + ASSERT_TRUE(found); + ASSERT_EQ(0, slot); + leaf->remove_from(slot); + } + { + auto [slot, found] = leaf->lower_bound(oid); + ASSERT_FALSE(found); + } +} diff --git a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc new file mode 100644 index 000000000..da7422bcb --- /dev/null +++ b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc @@ -0,0 +1,1204 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- +// vim: ts=8 sw=2 smarttab + +#include <array> +#include <cstring> +#include <memory> +#include <set> +#include <sstream> +#include <vector> + +#include "crimson/common/log.h" +#include "crimson/os/seastore/onode_manager/staged-fltree/node.h" +#include "crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager.h" +#include "crimson/os/seastore/onode_manager/staged-fltree/node_layout.h" +#include "crimson/os/seastore/onode_manager/staged-fltree/tree.h" +#include "crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h" + +#include "test/crimson/gtest_seastar.h" +#include "test/crimson/seastore/transaction_manager_test_state.h" + +using namespace crimson::os::seastore::onode; + +namespace { + constexpr bool IS_DUMMY_SYNC = false; + + [[maybe_unused]] seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_test); + } + + ghobject_t make_ghobj( + shard_t shard, pool_t pool, crush_hash_t crush, + std::string ns, std::string oid, snap_t snap, gen_t gen) { + return ghobject_t{shard_id_t{shard}, pool, crush, ns, oid, snap, gen}; + } + + // return a key_view_t and its underlying memory buffer. + // the buffer needs to be freed manually. + std::pair<key_view_t, void*> build_key_view(const ghobject_t& hobj) { + key_hobj_t key_hobj(hobj); + size_t key_size = sizeof(shard_pool_crush_t) + sizeof(snap_gen_t) + + ns_oid_view_t::estimate_size<KeyT::HOBJ>(key_hobj); + void* p_mem = std::malloc(key_size); + + key_view_t key_view; + char* p_fill = (char*)p_mem + key_size; + + auto spc = shard_pool_crush_t::from_key<KeyT::HOBJ>(key_hobj); + p_fill -= sizeof(shard_pool_crush_t); + std::memcpy(p_fill, &spc, sizeof(shard_pool_crush_t)); + key_view.set(*reinterpret_cast<const shard_pool_crush_t*>(p_fill)); + + auto p_ns_oid = p_fill; + ns_oid_view_t::test_append<KeyT::HOBJ>(key_hobj, p_fill); + ns_oid_view_t ns_oid_view(p_ns_oid); + key_view.set(ns_oid_view); + + auto sg = snap_gen_t::from_key<KeyT::HOBJ>(key_hobj); + p_fill -= sizeof(snap_gen_t); + ceph_assert(p_fill == (char*)p_mem); + std::memcpy(p_fill, &sg, sizeof(snap_gen_t)); + key_view.set(*reinterpret_cast<const snap_gen_t*>(p_fill)); + + return {key_view, p_mem}; + } +} + +struct a_basic_test_t : public seastar_test_suite_t {}; + +TEST_F(a_basic_test_t, 1_basic_sizes) +{ + logger().info("\n" + "Bytes of struct:\n" + " node_header_t: {}\n" + " shard_pool_t: {}\n" + " shard_pool_crush_t: {}\n" + " crush_t: {}\n" + " snap_gen_t: {}\n" + " slot_0_t: {}\n" + " slot_1_t: {}\n" + " slot_3_t: {}\n" + " node_fields_0_t: {}\n" + " node_fields_1_t: {}\n" + " node_fields_2_t: {}\n" + " internal_fields_3_t: {}\n" + " leaf_fields_3_t: {}\n" + " internal_sub_item_t: {}", + sizeof(node_header_t), sizeof(shard_pool_t), + sizeof(shard_pool_crush_t), sizeof(crush_t), sizeof(snap_gen_t), + sizeof(slot_0_t), sizeof(slot_1_t), sizeof(slot_3_t), + sizeof(node_fields_0_t), sizeof(node_fields_1_t), sizeof(node_fields_2_t), + sizeof(internal_fields_3_t), sizeof(leaf_fields_3_t), sizeof(internal_sub_item_t) + ); + + auto hobj = make_ghobj(0, 0, 0, "n", "o", 0, 0); + key_hobj_t key(hobj); + auto [key_view, p_mem] = build_key_view(hobj); + onode_t value = {2}; +#define _STAGE_T(NodeType) node_to_stage_t<typename NodeType::node_stage_t> +#define NXT_T(StageType) staged<typename StageType::next_param_t> + laddr_packed_t i_value{0}; + logger().info("\n" + "Bytes of a key-value insertion (full-string):\n" + " s-p-c, 'n'-'o', s-g => onode_t(2): typically internal 41B, leaf 35B\n" + " InternalNode0: {} {} {}\n" + " InternalNode1: {} {} {}\n" + " InternalNode2: {} {}\n" + " InternalNode3: {}\n" + " LeafNode0: {} {} {}\n" + " LeafNode1: {} {} {}\n" + " LeafNode2: {} {}\n" + " LeafNode3: {}", + _STAGE_T(InternalNode0)::template insert_size<KeyT::VIEW>(key_view, i_value), + NXT_T(_STAGE_T(InternalNode0))::template insert_size<KeyT::VIEW>(key_view, i_value), + NXT_T(NXT_T(_STAGE_T(InternalNode0)))::template insert_size<KeyT::VIEW>(key_view, i_value), + _STAGE_T(InternalNode1)::template insert_size<KeyT::VIEW>(key_view, i_value), + NXT_T(_STAGE_T(InternalNode1))::template insert_size<KeyT::VIEW>(key_view, i_value), + NXT_T(NXT_T(_STAGE_T(InternalNode1)))::template insert_size<KeyT::VIEW>(key_view, i_value), + _STAGE_T(InternalNode2)::template insert_size<KeyT::VIEW>(key_view, i_value), + NXT_T(_STAGE_T(InternalNode2))::template insert_size<KeyT::VIEW>(key_view, i_value), + _STAGE_T(InternalNode3)::template insert_size<KeyT::VIEW>(key_view, i_value), + _STAGE_T(LeafNode0)::template insert_size<KeyT::HOBJ>(key, value), + NXT_T(_STAGE_T(LeafNode0))::template insert_size<KeyT::HOBJ>(key, value), + NXT_T(NXT_T(_STAGE_T(LeafNode0)))::template insert_size<KeyT::HOBJ>(key, value), + _STAGE_T(LeafNode1)::template insert_size<KeyT::HOBJ>(key, value), + NXT_T(_STAGE_T(LeafNode1))::template insert_size<KeyT::HOBJ>(key, value), + NXT_T(NXT_T(_STAGE_T(LeafNode1)))::template insert_size<KeyT::HOBJ>(key, value), + _STAGE_T(LeafNode2)::template insert_size<KeyT::HOBJ>(key, value), + NXT_T(_STAGE_T(LeafNode2))::template insert_size<KeyT::HOBJ>(key, value), + _STAGE_T(LeafNode3)::template insert_size<KeyT::HOBJ>(key, value) + ); + std::free(p_mem); +} + +TEST_F(a_basic_test_t, 2_node_sizes) +{ + run_async([this] { + auto nm = NodeExtentManager::create_dummy(IS_DUMMY_SYNC); + auto t = make_transaction(); + context_t c{*nm, *t}; + std::array<std::pair<NodeImplURef, NodeExtentMutable>, 16> nodes = { + InternalNode0::allocate(c, false, 1u).unsafe_get0().make_pair(), + InternalNode1::allocate(c, false, 1u).unsafe_get0().make_pair(), + InternalNode2::allocate(c, false, 1u).unsafe_get0().make_pair(), + InternalNode3::allocate(c, false, 1u).unsafe_get0().make_pair(), + InternalNode0::allocate(c, true, 1u).unsafe_get0().make_pair(), + InternalNode1::allocate(c, true, 1u).unsafe_get0().make_pair(), + InternalNode2::allocate(c, true, 1u).unsafe_get0().make_pair(), + InternalNode3::allocate(c, true, 1u).unsafe_get0().make_pair(), + LeafNode0::allocate(c, false, 0u).unsafe_get0().make_pair(), + LeafNode1::allocate(c, false, 0u).unsafe_get0().make_pair(), + LeafNode2::allocate(c, false, 0u).unsafe_get0().make_pair(), + LeafNode3::allocate(c, false, 0u).unsafe_get0().make_pair(), + LeafNode0::allocate(c, true, 0u).unsafe_get0().make_pair(), + LeafNode1::allocate(c, true, 0u).unsafe_get0().make_pair(), + LeafNode2::allocate(c, true, 0u).unsafe_get0().make_pair(), + LeafNode3::allocate(c, true, 0u).unsafe_get0().make_pair() + }; + std::ostringstream oss; + oss << "\nallocated nodes:"; + for (auto iter = nodes.begin(); iter != nodes.end(); ++iter) { + oss << "\n "; + auto& ref_node = iter->first; + ref_node->dump_brief(oss); + } + logger().info("{}", oss.str()); + }); +} + +struct b_dummy_tree_test_t : public seastar_test_suite_t { + NodeExtentManagerURef moved_nm; + TransactionRef ref_t; + Transaction& t; + context_t c; + Btree tree; + + b_dummy_tree_test_t() + : moved_nm{NodeExtentManager::create_dummy(IS_DUMMY_SYNC)}, + ref_t{make_transaction()}, + t{*ref_t}, + c{*moved_nm, t}, + tree{std::move(moved_nm)} {} + + seastar::future<> set_up_fut() override final { + return tree.mkfs(t).handle_error( + crimson::ct_error::all_same_way([] { + ASSERT_FALSE("Unable to mkfs"); + }) + ); + } +}; + +TEST_F(b_dummy_tree_test_t, 3_random_insert_leaf_node) +{ + run_async([this] { + logger().info("\n---------------------------------------------" + "\nrandomized leaf node insert:\n"); + auto key_s = make_ghobj(0, 0, 0, "ns", "oid", 0, 0); + auto key_e = make_ghobj( + std::numeric_limits<shard_t>::max(), 0, 0, "ns", "oid", 0, 0); + ASSERT_TRUE(tree.find(t, key_s).unsafe_get0().is_end()); + ASSERT_TRUE(tree.begin(t).unsafe_get0().is_end()); + ASSERT_TRUE(tree.last(t).unsafe_get0().is_end()); + + std::vector<std::tuple<ghobject_t, + const onode_t*, + Btree::Cursor>> insert_history; + auto f_validate_insert_new = [this, &insert_history] ( + const ghobject_t& key, const onode_t& value) { + auto [cursor, success] = tree.insert(t, key, value).unsafe_get0(); + ceph_assert(success); + insert_history.emplace_back(key, &value, cursor); + Onodes::validate_cursor(cursor, key, value); + auto cursor_ = tree.lower_bound(t, key).unsafe_get0(); + ceph_assert(cursor_.get_ghobj() == key); + ceph_assert(cursor_.value() == cursor.value()); + return cursor.value(); + }; + auto onodes = Onodes(15); + + // insert key1, onode1 at STAGE_LEFT + auto key1 = make_ghobj(3, 3, 3, "ns3", "oid3", 3, 3); + auto& onode1 = onodes.pick(); + auto p_value1 = f_validate_insert_new(key1, onode1); + + // validate lookup + { + auto cursor1_s = tree.lower_bound(t, key_s).unsafe_get0(); + ASSERT_EQ(cursor1_s.get_ghobj(), key1); + ASSERT_EQ(cursor1_s.value(), p_value1); + auto cursor1_e = tree.lower_bound(t, key_e).unsafe_get0(); + ASSERT_TRUE(cursor1_e.is_end()); + } + + // insert the same key1 with a different onode + { + auto& onode1_dup = onodes.pick(); + auto [cursor1_dup, ret1_dup] = tree.insert( + t, key1, onode1_dup).unsafe_get0(); + ASSERT_FALSE(ret1_dup); + Onodes::validate_cursor(cursor1_dup, key1, onode1); + } + + // insert key2, onode2 to key1's left at STAGE_LEFT + // insert node front at STAGE_LEFT + auto key2 = make_ghobj(2, 2, 2, "ns3", "oid3", 3, 3); + auto& onode2 = onodes.pick(); + f_validate_insert_new(key2, onode2); + + // insert key3, onode3 to key1's right at STAGE_LEFT + // insert node last at STAGE_LEFT + auto key3 = make_ghobj(4, 4, 4, "ns3", "oid3", 3, 3); + auto& onode3 = onodes.pick(); + f_validate_insert_new(key3, onode3); + + // insert key4, onode4 to key1's left at STAGE_STRING (collision) + auto key4 = make_ghobj(3, 3, 3, "ns2", "oid2", 3, 3); + auto& onode4 = onodes.pick(); + f_validate_insert_new(key4, onode4); + + // insert key5, onode5 to key1's right at STAGE_STRING (collision) + auto key5 = make_ghobj(3, 3, 3, "ns4", "oid4", 3, 3); + auto& onode5 = onodes.pick(); + f_validate_insert_new(key5, onode5); + + // insert key6, onode6 to key1's left at STAGE_RIGHT + auto key6 = make_ghobj(3, 3, 3, "ns3", "oid3", 2, 2); + auto& onode6 = onodes.pick(); + f_validate_insert_new(key6, onode6); + + // insert key7, onode7 to key1's right at STAGE_RIGHT + auto key7 = make_ghobj(3, 3, 3, "ns3", "oid3", 4, 4); + auto& onode7 = onodes.pick(); + f_validate_insert_new(key7, onode7); + + // insert node front at STAGE_RIGHT + auto key8 = make_ghobj(2, 2, 2, "ns3", "oid3", 2, 2); + auto& onode8 = onodes.pick(); + f_validate_insert_new(key8, onode8); + + // insert node front at STAGE_STRING (collision) + auto key9 = make_ghobj(2, 2, 2, "ns2", "oid2", 3, 3); + auto& onode9 = onodes.pick(); + f_validate_insert_new(key9, onode9); + + // insert node last at STAGE_RIGHT + auto key10 = make_ghobj(4, 4, 4, "ns3", "oid3", 4, 4); + auto& onode10 = onodes.pick(); + f_validate_insert_new(key10, onode10); + + // insert node last at STAGE_STRING (collision) + auto key11 = make_ghobj(4, 4, 4, "ns4", "oid4", 3, 3); + auto& onode11 = onodes.pick(); + f_validate_insert_new(key11, onode11); + + // insert key, value randomly until a perfect 3-ary tree is formed + std::vector<std::pair<ghobject_t, const onode_t*>> kvs{ + {make_ghobj(2, 2, 2, "ns2", "oid2", 2, 2), &onodes.pick()}, + {make_ghobj(2, 2, 2, "ns2", "oid2", 4, 4), &onodes.pick()}, + {make_ghobj(2, 2, 2, "ns3", "oid3", 4, 4), &onodes.pick()}, + {make_ghobj(2, 2, 2, "ns4", "oid4", 2, 2), &onodes.pick()}, + {make_ghobj(2, 2, 2, "ns4", "oid4", 3, 3), &onodes.pick()}, + {make_ghobj(2, 2, 2, "ns4", "oid4", 4, 4), &onodes.pick()}, + {make_ghobj(3, 3, 3, "ns2", "oid2", 2, 2), &onodes.pick()}, + {make_ghobj(3, 3, 3, "ns2", "oid2", 4, 4), &onodes.pick()}, + {make_ghobj(3, 3, 3, "ns4", "oid4", 2, 2), &onodes.pick()}, + {make_ghobj(3, 3, 3, "ns4", "oid4", 4, 4), &onodes.pick()}, + {make_ghobj(4, 4, 4, "ns2", "oid2", 2, 2), &onodes.pick()}, + {make_ghobj(4, 4, 4, "ns2", "oid2", 3, 3), &onodes.pick()}, + {make_ghobj(4, 4, 4, "ns2", "oid2", 4, 4), &onodes.pick()}, + {make_ghobj(4, 4, 4, "ns3", "oid3", 2, 2), &onodes.pick()}, + {make_ghobj(4, 4, 4, "ns4", "oid4", 2, 2), &onodes.pick()}, + {make_ghobj(4, 4, 4, "ns4", "oid4", 4, 4), &onodes.pick()}}; + auto [smallest_key, smallest_value] = kvs[0]; + auto [largest_key, largest_value] = kvs[kvs.size() - 1]; + std::random_shuffle(kvs.begin(), kvs.end()); + std::for_each(kvs.begin(), kvs.end(), [&f_validate_insert_new] (auto& kv) { + f_validate_insert_new(kv.first, *kv.second); + }); + ASSERT_EQ(tree.height(t).unsafe_get0(), 1); + ASSERT_FALSE(tree.test_is_clean()); + + for (auto& [k, v, c] : insert_history) { + // validate values in tree keep intact + auto cursor = tree.lower_bound(t, k).unsafe_get0(); + Onodes::validate_cursor(cursor, k, *v); + // validate values in cursors keep intact + Onodes::validate_cursor(c, k, *v); + } + Onodes::validate_cursor( + tree.lower_bound(t, key_s).unsafe_get0(), smallest_key, *smallest_value); + Onodes::validate_cursor( + tree.begin(t).unsafe_get0(), smallest_key, *smallest_value); + Onodes::validate_cursor( + tree.last(t).unsafe_get0(), largest_key, *largest_value); + + std::ostringstream oss; + tree.dump(t, oss); + logger().info("\n{}\n", oss.str()); + + insert_history.clear(); + }); +} + +static std::set<ghobject_t> build_key_set( + std::pair<unsigned, unsigned> range_2, + std::pair<unsigned, unsigned> range_1, + std::pair<unsigned, unsigned> range_0, + std::string padding = "", + bool is_internal = false) { + ceph_assert(range_1.second <= 10); + std::set<ghobject_t> ret; + ghobject_t key; + for (unsigned i = range_2.first; i < range_2.second; ++i) { + for (unsigned j = range_1.first; j < range_1.second; ++j) { + for (unsigned k = range_0.first; k < range_0.second; ++k) { + std::ostringstream os_ns; + os_ns << "ns" << j; + std::ostringstream os_oid; + os_oid << "oid" << j << padding; + key = make_ghobj(i, i, i, os_ns.str(), os_oid.str(), k, k); + ret.insert(key); + } + } + } + if (is_internal) { + ret.insert(make_ghobj(9, 9, 9, "ns~last", "oid~last", 9, 9)); + } + return ret; +} + +class TestTree { + public: + TestTree() + : moved_nm{NodeExtentManager::create_dummy(IS_DUMMY_SYNC)}, + ref_t{make_transaction()}, + t{*ref_t}, + c{*moved_nm, t}, + tree{std::move(moved_nm)}, + onodes{0} {} + + seastar::future<> build_tree( + std::pair<unsigned, unsigned> range_2, + std::pair<unsigned, unsigned> range_1, + std::pair<unsigned, unsigned> range_0, + size_t onode_size) { + return seastar::async([this, range_2, range_1, range_0, onode_size] { + tree.mkfs(t).unsafe_get0(); + //logger().info("\n---------------------------------------------" + // "\nbefore leaf node split:\n"); + auto keys = build_key_set(range_2, range_1, range_0); + for (auto& key : keys) { + auto& value = onodes.create(onode_size); + insert_tree(key, value).get0(); + } + ASSERT_EQ(tree.height(t).unsafe_get0(), 1); + ASSERT_FALSE(tree.test_is_clean()); + //std::ostringstream oss; + //tree.dump(t, oss); + //logger().info("\n{}\n", oss.str()); + }); + } + + seastar::future<> build_tree( + const std::vector<ghobject_t>& keys, const std::vector<const onode_t*>& values) { + return seastar::async([this, keys, values] { + tree.mkfs(t).unsafe_get0(); + //logger().info("\n---------------------------------------------" + // "\nbefore leaf node split:\n"); + ASSERT_EQ(keys.size(), values.size()); + auto key_iter = keys.begin(); + auto value_iter = values.begin(); + while (key_iter != keys.end()) { + insert_tree(*key_iter, **value_iter).get0(); + ++key_iter; + ++value_iter; + } + ASSERT_EQ(tree.height(t).unsafe_get0(), 1); + ASSERT_FALSE(tree.test_is_clean()); + //std::ostringstream oss; + //tree.dump(t, oss); + //logger().info("\n{}\n", oss.str()); + }); + } + + seastar::future<> split(const ghobject_t& key, const onode_t& value, + const split_expectation_t& expected) { + return seastar::async([this, key, &value, expected] { + Btree tree_clone(NodeExtentManager::create_dummy(IS_DUMMY_SYNC)); + auto ref_t_clone = make_transaction(); + Transaction& t_clone = *ref_t_clone; + tree_clone.test_clone_from(t_clone, t, tree).unsafe_get0(); + + logger().info("insert {}:", key_hobj_t(key)); + auto [cursor, success] = tree_clone.insert(t_clone, key, value).unsafe_get0(); + ASSERT_TRUE(success); + Onodes::validate_cursor(cursor, key, value); + + std::ostringstream oss; + tree_clone.dump(t_clone, oss); + logger().info("dump new root:\n{}", oss.str()); + EXPECT_EQ(tree_clone.height(t_clone).unsafe_get0(), 2); + + for (auto& [k, v, c] : insert_history) { + auto result = tree_clone.lower_bound(t_clone, k).unsafe_get0(); + Onodes::validate_cursor(result, k, *v); + } + auto result = tree_clone.lower_bound(t_clone, key).unsafe_get0(); + Onodes::validate_cursor(result, key, value); + EXPECT_TRUE(last_split.match(expected)); + }); + } + + const onode_t& create_onode(size_t size) { + return onodes.create(size); + } + + private: + seastar::future<> insert_tree(const ghobject_t& key, const onode_t& value) { + return seastar::async([this, &key, &value] { + auto [cursor, success] = tree.insert(t, key, value).unsafe_get0(); + ASSERT_TRUE(success); + Onodes::validate_cursor(cursor, key, value); + insert_history.emplace_back(key, &value, cursor); + }); + } + + NodeExtentManagerURef moved_nm; + TransactionRef ref_t; + Transaction& t; + context_t c; + Btree tree; + Onodes onodes; + std::vector<std::tuple< + ghobject_t, const onode_t*, Btree::Cursor>> insert_history; +}; + +struct c_dummy_test_t : public seastar_test_suite_t {}; + +TEST_F(c_dummy_test_t, 4_split_leaf_node) +{ + run_async([this] { + { + TestTree test; + test.build_tree({2, 5}, {2, 5}, {2, 5}, 120).get0(); + + auto& onode = test.create_onode(1144); + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to left front at stage 2, 1, 0\n"); + test.split(make_ghobj(1, 1, 1, "ns3", "oid3", 3, 3), onode, + {2u, 2u, true, InsertType::BEGIN}).get0(); + test.split(make_ghobj(2, 2, 2, "ns1", "oid1", 3, 3), onode, + {2u, 1u, true, InsertType::BEGIN}).get0(); + test.split(make_ghobj(2, 2, 2, "ns2", "oid2", 1, 1), onode, + {2u, 0u, true, InsertType::BEGIN}).get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to left back at stage 0, 1, 2, 1, 0\n"); + test.split(make_ghobj(2, 2, 2, "ns4", "oid4", 5, 5), onode, + {2u, 0u, true, InsertType::LAST}).get0(); + test.split(make_ghobj(2, 2, 2, "ns5", "oid5", 3, 3), onode, + {2u, 1u, true, InsertType::LAST}).get0(); + test.split(make_ghobj(2, 3, 3, "ns3", "oid3", 3, 3), onode, + {2u, 2u, true, InsertType::LAST}).get0(); + test.split(make_ghobj(3, 3, 3, "ns1", "oid1", 3, 3), onode, + {2u, 1u, true, InsertType::LAST}).get0(); + test.split(make_ghobj(3, 3, 3, "ns2", "oid2", 1, 1), onode, + {2u, 0u, true, InsertType::LAST}).get0(); + + auto& onode0 = test.create_onode(1416); + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to right front at stage 0, 1, 2, 1, 0\n"); + test.split(make_ghobj(3, 3, 3, "ns4", "oid4", 5, 5), onode0, + {2u, 0u, false, InsertType::BEGIN}).get0(); + test.split(make_ghobj(3, 3, 3, "ns5", "oid5", 3, 3), onode0, + {2u, 1u, false, InsertType::BEGIN}).get0(); + test.split(make_ghobj(3, 4, 4, "ns3", "oid3", 3, 3), onode0, + {2u, 2u, false, InsertType::BEGIN}).get0(); + test.split(make_ghobj(4, 4, 4, "ns1", "oid1", 3, 3), onode0, + {2u, 1u, false, InsertType::BEGIN}).get0(); + test.split(make_ghobj(4, 4, 4, "ns2", "oid2", 1, 1), onode0, + {2u, 0u, false, InsertType::BEGIN}).get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to right back at stage 0, 1, 2\n"); + test.split(make_ghobj(4, 4, 4, "ns4", "oid4", 5, 5), onode0, + {2u, 0u, false, InsertType::LAST}).get0(); + test.split(make_ghobj(4, 4, 4, "ns5", "oid5", 3, 3), onode0, + {2u, 1u, false, InsertType::LAST}).get0(); + test.split(make_ghobj(5, 5, 5, "ns3", "oid3", 3, 3), onode0, + {2u, 2u, false, InsertType::LAST}).get0(); + + auto& onode1 = test.create_onode(316); + logger().info("\n---------------------------------------------" + "\nsplit at stage 1; insert to left middle at stage 0, 1, 2, 1, 0\n"); + test.split(make_ghobj(2, 2, 2, "ns4", "oid4", 5, 5), onode1, + {1u, 0u, true, InsertType::MID}).get0(); + test.split(make_ghobj(2, 2, 2, "ns5", "oid5", 3, 3), onode1, + {1u, 1u, true, InsertType::MID}).get0(); + test.split(make_ghobj(2, 2, 3, "ns3", "oid3", 3, 3), onode1, + {1u, 2u, true, InsertType::MID}).get0(); + test.split(make_ghobj(3, 3, 3, "ns1", "oid1", 3, 3), onode1, + {1u, 1u, true, InsertType::MID}).get0(); + test.split(make_ghobj(3, 3, 3, "ns2", "oid2", 1, 1), onode1, + {1u, 0u, true, InsertType::MID}).get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 1; insert to left back at stage 0, 1, 0\n"); + test.split(make_ghobj(3, 3, 3, "ns2", "oid2", 5, 5), onode1, + {1u, 0u, true, InsertType::LAST}).get0(); + test.split(make_ghobj(3, 3, 3, "ns2", "oid3", 3, 3), onode1, + {1u, 1u, true, InsertType::LAST}).get0(); + test.split(make_ghobj(3, 3, 3, "ns3", "oid3", 1, 1), onode1, + {1u, 0u, true, InsertType::LAST}).get0(); + + auto& onode2 = test.create_onode(452); + logger().info("\n---------------------------------------------" + "\nsplit at stage 1; insert to right front at stage 0, 1, 0\n"); + test.split(make_ghobj(3, 3, 3, "ns3", "oid3", 5, 5), onode2, + {1u, 0u, false, InsertType::BEGIN}).get0(); + test.split(make_ghobj(3, 3, 3, "ns3", "oid4", 3, 3), onode2, + {1u, 1u, false, InsertType::BEGIN}).get0(); + test.split(make_ghobj(3, 3, 3, "ns4", "oid4", 1, 1), onode2, + {1u, 0u, false, InsertType::BEGIN}).get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 1; insert to right middle at stage 0, 1, 2, 1, 0\n"); + test.split(make_ghobj(3, 3, 3, "ns4", "oid4", 5, 5), onode2, + {1u, 0u, false, InsertType::MID}).get0(); + test.split(make_ghobj(3, 3, 3, "ns5", "oid5", 3, 3), onode2, + {1u, 1u, false, InsertType::MID}).get0(); + test.split(make_ghobj(3, 3, 4, "ns3", "oid3", 3, 3), onode2, + {1u, 2u, false, InsertType::MID}).get0(); + test.split(make_ghobj(4, 4, 4, "ns1", "oid1", 3, 3), onode2, + {1u, 1u, false, InsertType::MID}).get0(); + test.split(make_ghobj(4, 4, 4, "ns2", "oid2", 1, 1), onode2, + {1u, 0u, false, InsertType::MID}).get0(); + + auto& onode3 = test.create_onode(834); + logger().info("\n---------------------------------------------" + "\nsplit at stage 0; insert to right middle at stage 0, 1, 2, 1, 0\n"); + test.split(make_ghobj(3, 3, 3, "ns4", "oid4", 5, 5), onode3, + {0u, 0u, false, InsertType::MID}).get0(); + test.split(make_ghobj(3, 3, 3, "ns5", "oid5", 3, 3), onode3, + {0u, 1u, false, InsertType::MID}).get0(); + test.split(make_ghobj(3, 3, 4, "ns3", "oid3", 3, 3), onode3, + {0u, 2u, false, InsertType::MID}).get0(); + test.split(make_ghobj(4, 4, 4, "ns1", "oid1", 3, 3), onode3, + {0u, 1u, false, InsertType::MID}).get0(); + test.split(make_ghobj(4, 4, 4, "ns2", "oid2", 1, 1), onode3, + {0u, 0u, false, InsertType::MID}).get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 0; insert to right front at stage 0\n"); + test.split(make_ghobj(3, 3, 3, "ns4", "oid4", 2, 3), onode3, + {0u, 0u, false, InsertType::BEGIN}).get0(); + + auto& onode4 = test.create_onode(572); + logger().info("\n---------------------------------------------" + "\nsplit at stage 0; insert to left back at stage 0\n"); + test.split(make_ghobj(3, 3, 3, "ns2", "oid2", 3, 4), onode4, + {0u, 0u, true, InsertType::LAST}).get0(); + } + + { + TestTree test; + test.build_tree({2, 4}, {2, 4}, {2, 4}, 232).get0(); + auto& onode = test.create_onode(1996); + logger().info("\n---------------------------------------------" + "\nsplit at [0, 0, 0]; insert to left front at stage 2, 1, 0\n"); + test.split(make_ghobj(1, 1, 1, "ns3", "oid3", 3, 3), onode, + {2u, 2u, true, InsertType::BEGIN}).get0(); + EXPECT_TRUE(last_split.match_split_pos({0, {0, {0}}})); + test.split(make_ghobj(2, 2, 2, "ns1", "oid1", 3, 3), onode, + {2u, 1u, true, InsertType::BEGIN}).get0(); + EXPECT_TRUE(last_split.match_split_pos({0, {0, {0}}})); + test.split(make_ghobj(2, 2, 2, "ns2", "oid2", 1, 1), onode, + {2u, 0u, true, InsertType::BEGIN}).get0(); + EXPECT_TRUE(last_split.match_split_pos({0, {0, {0}}})); + } + + { + TestTree test; + std::vector<ghobject_t> keys = { + make_ghobj(2, 2, 2, "ns3", "oid3", 3, 3), + make_ghobj(3, 3, 3, "ns3", "oid3", 3, 3)}; + std::vector<const onode_t*> values = { + &test.create_onode(1360), + &test.create_onode(1632)}; + test.build_tree(keys, values).get0(); + auto& onode = test.create_onode(1640); + logger().info("\n---------------------------------------------" + "\nsplit at [END, END, END]; insert to right at stage 0, 1, 2\n"); + test.split(make_ghobj(3, 3, 3, "ns3", "oid3", 4, 4), onode, + {0u, 0u, false, InsertType::BEGIN}).get0(); + EXPECT_TRUE(last_split.match_split_pos({1, {0, {1}}})); + test.split(make_ghobj(3, 3, 3, "ns4", "oid4", 3, 3), onode, + {1u, 1u, false, InsertType::BEGIN}).get0(); + EXPECT_TRUE(last_split.match_split_pos({1, {1, {0}}})); + test.split(make_ghobj(4, 4, 4, "ns3", "oid3", 3, 3), onode, + {2u, 2u, false, InsertType::BEGIN}).get0(); + EXPECT_TRUE(last_split.match_split_pos({2, {0, {0}}})); + } + }); +} + +namespace crimson::os::seastore::onode { + +class DummyChildPool { + class DummyChildImpl final : public NodeImpl { + public: + using URef = std::unique_ptr<DummyChildImpl>; + DummyChildImpl(const std::set<ghobject_t>& keys, bool is_level_tail, laddr_t laddr) + : keys{keys}, _is_level_tail{is_level_tail}, _laddr{laddr} { + std::tie(key_view, p_mem_key_view) = build_key_view(*keys.crbegin()); + } + ~DummyChildImpl() override { + std::free(p_mem_key_view); + } + + const std::set<ghobject_t>& get_keys() const { return keys; } + + void reset(const std::set<ghobject_t>& _keys, bool level_tail) { + keys = _keys; + _is_level_tail = level_tail; + std::free(p_mem_key_view); + std::tie(key_view, p_mem_key_view) = build_key_view(*keys.crbegin()); + } + + public: + laddr_t laddr() const override { return _laddr; } + bool is_level_tail() const override { return _is_level_tail; } + + protected: + field_type_t field_type() const override { return field_type_t::N0; } + level_t level() const override { return 0u; } + key_view_t get_largest_key_view() const override { return key_view; } + void prepare_mutate(context_t) override { + ceph_abort("impossible path"); } + bool is_empty() const override { + ceph_abort("impossible path"); } + node_offset_t free_size() const override { + ceph_abort("impossible path"); } + key_view_t get_key_view(const search_position_t&) const override { + ceph_abort("impossible path"); } + void next_position(search_position_t&) const override { + ceph_abort("impossible path"); } + node_stats_t get_stats() const override { + ceph_abort("impossible path"); } + std::ostream& dump(std::ostream&) const override { + ceph_abort("impossible path"); } + std::ostream& dump_brief(std::ostream&) const override { + ceph_abort("impossible path"); } + void validate_layout() const override { + ceph_abort("impossible path"); } + void test_copy_to(NodeExtentMutable&) const override { + ceph_abort("impossible path"); } + void test_set_tail(NodeExtentMutable&) override { + ceph_abort("impossible path"); } + + private: + std::set<ghobject_t> keys; + bool _is_level_tail; + laddr_t _laddr; + + key_view_t key_view; + void* p_mem_key_view; + }; + + class DummyChild final : public Node { + public: + ~DummyChild() override = default; + + node_future<> populate_split( + context_t c, std::set<Ref<DummyChild>>& splitable_nodes) { + ceph_assert(can_split()); + ceph_assert(splitable_nodes.find(this) != splitable_nodes.end()); + + size_t index; + const auto& keys = impl->get_keys(); + if (keys.size() == 2) { + index = 1; + } else { + index = rd() % (keys.size() - 2) + 1; + } + auto iter = keys.begin(); + std::advance(iter, index); + + std::set<ghobject_t> left_keys(keys.begin(), iter); + std::set<ghobject_t> right_keys(iter, keys.end()); + bool right_is_tail = impl->is_level_tail(); + impl->reset(left_keys, false); + auto right_child = DummyChild::create_new(right_keys, right_is_tail, pool); + if (!can_split()) { + splitable_nodes.erase(this); + } + if (right_child->can_split()) { + splitable_nodes.insert(right_child); + } + return insert_parent(c, right_child); + } + + node_future<> insert_and_split( + context_t c, const ghobject_t& insert_key, + std::set<Ref<DummyChild>>& splitable_nodes) { + const auto& keys = impl->get_keys(); + ceph_assert(keys.size() == 1); + auto& key = *keys.begin(); + ceph_assert(insert_key < key); + + std::set<ghobject_t> new_keys; + new_keys.insert(insert_key); + new_keys.insert(key); + impl->reset(new_keys, impl->is_level_tail()); + + splitable_nodes.clear(); + splitable_nodes.insert(this); + auto fut = populate_split(c, splitable_nodes); + ceph_assert(splitable_nodes.size() == 0); + return fut; + } + + bool match_pos(const search_position_t& pos) const { + ceph_assert(!is_root()); + return pos == parent_info().position; + } + + static Ref<DummyChild> create( + const std::set<ghobject_t>& keys, bool is_level_tail, + laddr_t addr, DummyChildPool& pool) { + auto ref_impl = std::make_unique<DummyChildImpl>(keys, is_level_tail, addr); + return new DummyChild(ref_impl.get(), std::move(ref_impl), pool); + } + + static Ref<DummyChild> create_new( + const std::set<ghobject_t>& keys, bool is_level_tail, DummyChildPool& pool) { + static laddr_t seed = 0; + return create(keys, is_level_tail, seed++, pool); + } + + static node_future<Ref<DummyChild>> create_initial( + context_t c, const std::set<ghobject_t>& keys, + DummyChildPool& pool, RootNodeTracker& root_tracker) { + auto initial = create_new(keys, true, pool); + return c.nm.get_super(c.t, root_tracker + ).safe_then([c, &pool, initial](auto super) { + initial->make_root_new(c, std::move(super)); + return initial->upgrade_root(c).safe_then([initial] { + return initial; + }); + }); + } + + protected: + node_future<> test_clone_non_root( + context_t, Ref<InternalNode> new_parent) const override { + ceph_assert(!is_root()); + auto p_pool_clone = pool.pool_clone_in_progress; + ceph_assert(p_pool_clone != nullptr); + auto clone = create( + impl->get_keys(), impl->is_level_tail(), impl->laddr(), *p_pool_clone); + clone->as_child(parent_info().position, new_parent); + return node_ertr::now(); + } + node_future<Ref<tree_cursor_t>> lookup_smallest(context_t) override { + ceph_abort("impossible path"); } + node_future<Ref<tree_cursor_t>> lookup_largest(context_t) override { + ceph_abort("impossible path"); } + node_future<> test_clone_root(context_t, RootNodeTracker&) const override { + ceph_abort("impossible path"); } + node_future<search_result_t> lower_bound_tracked( + context_t, const key_hobj_t&, MatchHistory&) override { + ceph_abort("impossible path"); } + node_future<> do_get_tree_stats(context_t, tree_stats_t&) override { + ceph_abort("impossible path"); } + + private: + DummyChild(DummyChildImpl* impl, DummyChildImpl::URef&& ref, DummyChildPool& pool) + : Node(std::move(ref)), impl{impl}, pool{pool} { + pool.track_node(this); + } + + bool can_split() const { return impl->get_keys().size() > 1; } + + DummyChildImpl* impl; + DummyChildPool& pool; + mutable std::random_device rd; + }; + + public: + using node_ertr = Node::node_ertr; + template <class ValueT=void> + using node_future = Node::node_future<ValueT>; + + DummyChildPool() = default; + ~DummyChildPool() { reset(); } + + node_future<> build_tree(const std::set<ghobject_t>& keys) { + reset(); + + // create tree + auto ref_nm = NodeExtentManager::create_dummy(IS_DUMMY_SYNC); + p_nm = ref_nm.get(); + p_btree.emplace(std::move(ref_nm)); + return DummyChild::create_initial(get_context(), keys, *this, *p_btree->root_tracker + ).safe_then([this](auto initial_child) { + // split + splitable_nodes.insert(initial_child); + return crimson::do_until([this] { + if (splitable_nodes.empty()) { + return node_ertr::make_ready_future<bool>(true); + } + auto index = rd() % splitable_nodes.size(); + auto iter = splitable_nodes.begin(); + std::advance(iter, index); + Ref<DummyChild> child = *iter; + return child->populate_split(get_context(), splitable_nodes + ).safe_then([] { + return node_ertr::make_ready_future<bool>(false); + }); + }); + }).safe_then([this] { + //std::ostringstream oss; + //p_btree->dump(t(), oss); + //logger().info("\n{}\n", oss.str()); + return p_btree->height(t()); + }).safe_then([](auto height) { + ceph_assert(height == 2); + }); + } + + seastar::future<> test_split(ghobject_t key, search_position_t pos, + const split_expectation_t& expected) { + return seastar::async([this, key, pos, expected] { + logger().info("insert {} at {}:", key_hobj_t(key), pos); + DummyChildPool pool_clone; + pool_clone_in_progress = &pool_clone; + auto ref_nm = NodeExtentManager::create_dummy(IS_DUMMY_SYNC); + pool_clone.p_nm = ref_nm.get(); + pool_clone.p_btree.emplace(std::move(ref_nm)); + pool_clone.p_btree->test_clone_from( + pool_clone.t(), t(), *p_btree).unsafe_get0(); + pool_clone_in_progress = nullptr; + auto node_to_split = pool_clone.get_node_by_pos(pos); + node_to_split->insert_and_split( + pool_clone.get_context(), key, pool_clone.splitable_nodes).unsafe_get0(); + std::ostringstream oss; + pool_clone.p_btree->dump(pool_clone.t(), oss); + logger().info("dump new root:\n{}", oss.str()); + EXPECT_EQ(pool_clone.p_btree->height(pool_clone.t()).unsafe_get0(), 3); + EXPECT_TRUE(last_split.match(expected)); + }); + } + + private: + void reset() { + ceph_assert(pool_clone_in_progress == nullptr); + if (tracked_children.size()) { + ceph_assert(!p_btree->test_is_clean()); + tracked_children.clear(); + ceph_assert(p_btree->test_is_clean()); + p_nm = nullptr; + p_btree.reset(); + } else { + ceph_assert(!p_btree.has_value()); + } + splitable_nodes.clear(); + } + + void track_node(Ref<DummyChild> node) { + ceph_assert(tracked_children.find(node) == tracked_children.end()); + tracked_children.insert(node); + } + + Ref<DummyChild> get_node_by_pos(const search_position_t& pos) const { + auto iter = std::find_if( + tracked_children.begin(), tracked_children.end(), [&pos](auto& child) { + return child->match_pos(pos); + }); + ceph_assert(iter != tracked_children.end()); + return *iter; + } + + context_t get_context() { + ceph_assert(p_nm != nullptr); + return {*p_nm, t()}; + } + + Transaction& t() const { return *ref_t; } + + std::set<Ref<DummyChild>> tracked_children; + std::optional<Btree> p_btree; + NodeExtentManager* p_nm = nullptr; + TransactionRef ref_t = make_transaction(); + + std::random_device rd; + std::set<Ref<DummyChild>> splitable_nodes; + + DummyChildPool* pool_clone_in_progress = nullptr; +}; + +} + +TEST_F(c_dummy_test_t, 5_split_internal_node) +{ + run_async([this] { + DummyChildPool pool; + { + logger().info("\n---------------------------------------------" + "\nbefore internal node insert:\n"); + auto padding = std::string(250, '_'); + auto keys = build_key_set({2, 6}, {2, 5}, {2, 5}, padding, true); + keys.erase(make_ghobj(2, 2, 2, "ns2", "oid2" + padding, 2, 2)); + keys.erase(make_ghobj(2, 2, 2, "ns2", "oid2" + padding, 3, 3)); + keys.erase(make_ghobj(2, 2, 2, "ns2", "oid2" + padding, 4, 4)); + keys.erase(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 2, 2)); + keys.erase(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 3, 3)); + keys.erase(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 4, 4)); + auto padding_s = std::string(257, '_'); + keys.insert(make_ghobj(2, 2, 2, "ns2", "oid2" + padding_s, 2, 2)); + keys.insert(make_ghobj(2, 2, 2, "ns2", "oid2" + padding_s, 3, 3)); + keys.insert(make_ghobj(2, 2, 2, "ns2", "oid2" + padding_s, 4, 4)); + auto padding_e = std::string(248, '_'); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding_e, 2, 2)); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding_e, 3, 3)); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding_e, 4, 4)); + pool.build_tree(keys).unsafe_get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to right front at stage 0, 1, 2, 1, 0\n"); + pool.test_split(make_ghobj(3, 3, 3, "ns4", "oid4" + padding, 5, 5), {2, {0, {0}}}, + {2u, 0u, false, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(3, 3, 3, "ns5", "oid5", 3, 3), {2, {0, {0}}}, + {2u, 1u, false, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(3, 4, 4, "ns3", "oid3", 3, 3), {2, {0, {0}}}, + {2u, 2u, false, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(4, 4, 4, "ns1", "oid1", 3, 3), {2, {0, {0}}}, + {2u, 1u, false, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(4, 4, 4, "ns2", "oid2" + padding, 1, 1), {2, {0, {0}}}, + {2u, 0u, false, InsertType::BEGIN}).get(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to right middle at stage 0, 1, 2, 1, 0\n"); + pool.test_split(make_ghobj(4, 4, 4, "ns4", "oid4" + padding, 5, 5), {3, {0, {0}}}, + {2u, 0u, false, InsertType::MID}).get(); + pool.test_split(make_ghobj(4, 4, 4, "ns5", "oid5", 3, 3), {3, {0, {0}}}, + {2u, 1u, false, InsertType::MID}).get(); + pool.test_split(make_ghobj(4, 4, 5, "ns3", "oid3", 3, 3), {3, {0, {0}}}, + {2u, 2u, false, InsertType::MID}).get(); + pool.test_split(make_ghobj(5, 5, 5, "ns1", "oid1", 3, 3), {3, {0, {0}}}, + {2u, 1u, false, InsertType::MID}).get(); + pool.test_split(make_ghobj(5, 5, 5, "ns2", "oid2" + padding, 1, 1), {3, {0, {0}}}, + {2u, 0u, false, InsertType::MID}).get(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to right back at stage 0, 1, 2\n"); + pool.test_split(make_ghobj(5, 5, 5, "ns4", "oid4" + padding_e, 5, 5), search_position_t::end(), + {2u, 0u, false, InsertType::LAST}).get(); + pool.test_split(make_ghobj(5, 5, 5, "ns5", "oid5", 3, 3), search_position_t::end(), + {2u, 1u, false, InsertType::LAST}).get(); + pool.test_split(make_ghobj(6, 6, 6, "ns3", "oid3", 3, 3), search_position_t::end(), + {2u, 2u, false, InsertType::LAST}).get(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 0; insert to left front at stage 2, 1, 0\n"); + pool.test_split(make_ghobj(1, 1, 1, "ns3", "oid3", 3, 3), {0, {0, {0}}}, + {0u, 2u, true, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(2, 2, 2, "ns1", "oid1", 3, 3), {0, {0, {0}}}, + {0u, 1u, true, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(2, 2, 2, "ns2", "oid2" + padding_s, 1, 1), {0, {0, {0}}}, + {0u, 0u, true, InsertType::BEGIN}).get(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 0; insert to left middle at stage 0, 1, 2, 1, 0\n"); + pool.test_split(make_ghobj(2, 2, 2, "ns4", "oid4" + padding, 5, 5), {1, {0, {0}}}, + {0u, 0u, true, InsertType::MID}).get(); + pool.test_split(make_ghobj(2, 2, 2, "ns5", "oid5", 3, 3), {1, {0, {0}}}, + {0u, 1u, true, InsertType::MID}).get(); + pool.test_split(make_ghobj(2, 2, 3, "ns3", "oid3" + std::string(80, '_'), 3, 3), {1, {0, {0}}}, + {0u, 2u, true, InsertType::MID}).get(); + pool.test_split(make_ghobj(3, 3, 3, "ns1", "oid1", 3, 3), {1, {0, {0}}}, + {0u, 1u, true, InsertType::MID}).get(); + pool.test_split(make_ghobj(3, 3, 3, "ns2", "oid2" + padding, 1, 1), {1, {0, {0}}}, + {0u, 0u, true, InsertType::MID}).get(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 0; insert to left back at stage 0\n"); + pool.test_split(make_ghobj(3, 3, 3, "ns4", "oid4" + padding, 3, 4), {1, {2, {2}}}, + {0u, 0u, true, InsertType::LAST}).get(); + } + + { + logger().info("\n---------------------------------------------" + "\nbefore internal node insert (1):\n"); + auto padding = std::string(244, '_'); + auto keys = build_key_set({2, 6}, {2, 5}, {2, 5}, padding, true); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 5, 5)); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 6, 6)); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 7, 7)); + pool.build_tree(keys).unsafe_get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to left back at stage 0, 1, 2, 1\n"); + pool.test_split(make_ghobj(3, 3, 3, "ns4", "oid4" + padding, 5, 5), {2, {0, {0}}}, + {2u, 0u, true, InsertType::LAST}).get(); + pool.test_split(make_ghobj(3, 3, 3, "ns5", "oid5", 3, 3), {2, {0, {0}}}, + {2u, 1u, true, InsertType::LAST}).get(); + pool.test_split(make_ghobj(3, 4, 4, "n", "o", 3, 3), {2, {0, {0}}}, + {2u, 2u, true, InsertType::LAST}).get(); + pool.test_split(make_ghobj(4, 4, 4, "n", "o", 3, 3), {2, {0, {0}}}, + {2u, 1u, true, InsertType::LAST}).get(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to left middle at stage 2\n"); + pool.test_split(make_ghobj(2, 3, 3, "n", "o", 3, 3), {1, {0, {0}}}, + {2u, 2u, true, InsertType::MID}).get(); + } + + { + logger().info("\n---------------------------------------------" + "\nbefore internal node insert (2):\n"); + auto padding = std::string(243, '_'); + auto keys = build_key_set({2, 6}, {2, 5}, {2, 5}, padding, true); + keys.insert(make_ghobj(4, 4, 4, "n", "o", 3, 3)); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 5, 5)); + keys.insert(make_ghobj(5, 5, 5, "ns4", "oid4" + padding, 6, 6)); + pool.build_tree(keys).unsafe_get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 2; insert to left back at stage (0, 1, 2, 1,) 0\n"); + pool.test_split(make_ghobj(4, 4, 4, "n", "o", 2, 2), {2, {0, {0}}}, + {2u, 0u, true, InsertType::LAST}).get(); + } + + { + logger().info("\n---------------------------------------------" + "\nbefore internal node insert (3):\n"); + auto padding = std::string(420, '_'); + auto keys = build_key_set({2, 5}, {2, 5}, {2, 5}, padding, true); + keys.erase(make_ghobj(4, 4, 4, "ns4", "oid4" + padding, 2, 2)); + keys.erase(make_ghobj(4, 4, 4, "ns4", "oid4" + padding, 3, 3)); + keys.erase(make_ghobj(4, 4, 4, "ns4", "oid4" + padding, 4, 4)); + pool.build_tree(keys).unsafe_get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 1; insert to right front at stage 0, 1, 0\n"); + pool.test_split(make_ghobj(3, 3, 3, "ns2", "oid2" + padding, 5, 5), {1, {1, {0}}}, + {1u, 0u, false, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(3, 3, 3, "ns2", "oid3", 3, 3), {1, {1, {0}}}, + {1u, 1u, false, InsertType::BEGIN}).get(); + pool.test_split(make_ghobj(3, 3, 3, "ns3", "oid3" + padding, 1, 1), {1, {1, {0}}}, + {1u, 0u, false, InsertType::BEGIN}).get(); + } + + { + logger().info("\n---------------------------------------------" + "\nbefore internal node insert (4):\n"); + auto padding = std::string(361, '_'); + auto keys = build_key_set({2, 5}, {2, 5}, {2, 5}, padding, true); + keys.erase(make_ghobj(2, 2, 2, "ns2", "oid2" + padding, 2, 2)); + keys.erase(make_ghobj(2, 2, 2, "ns2", "oid2" + padding, 3, 3)); + keys.erase(make_ghobj(2, 2, 2, "ns2", "oid2" + padding, 4, 4)); + auto padding_s = std::string(387, '_'); + keys.insert(make_ghobj(2, 2, 2, "ns2", "oid2" + padding_s, 2, 2)); + keys.insert(make_ghobj(2, 2, 2, "ns2", "oid2" + padding_s, 3, 3)); + keys.insert(make_ghobj(2, 2, 2, "ns2", "oid2" + padding_s, 4, 4)); + pool.build_tree(keys).unsafe_get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 1; insert to left back at stage 0, 1\n"); + pool.test_split(make_ghobj(3, 3, 3, "ns2", "oid2" + padding, 5, 5), {1, {1, {0}}}, + {1u, 0u, true, InsertType::LAST}).get(); + pool.test_split(make_ghobj(3, 3, 3, "ns2", "oid3", 3, 3), {1, {1, {0}}}, + {1u, 1u, true, InsertType::LAST}).get(); + } + + { + logger().info("\n---------------------------------------------" + "\nbefore internal node insert (5):\n"); + auto padding = std::string(412, '_'); + auto keys = build_key_set({2, 5}, {2, 5}, {2, 5}, padding); + keys.insert(make_ghobj(3, 3, 3, "ns2", "oid3", 3, 3)); + keys.insert(make_ghobj(4, 4, 4, "ns3", "oid3" + padding, 5, 5)); + keys.insert(make_ghobj(9, 9, 9, "ns~last", "oid~last", 9, 9)); + keys.erase(make_ghobj(4, 4, 4, "ns4", "oid4" + padding, 2, 2)); + keys.erase(make_ghobj(4, 4, 4, "ns4", "oid4" + padding, 3, 3)); + keys.erase(make_ghobj(4, 4, 4, "ns4", "oid4" + padding, 4, 4)); + pool.build_tree(keys).unsafe_get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 1; insert to left back at stage (0, 1,) 0\n"); + pool.test_split(make_ghobj(3, 3, 3, "ns2", "oid3", 2, 2), {1, {1, {0}}}, + {1u, 0u, true, InsertType::LAST}).get(); + } + + { + logger().info("\n---------------------------------------------" + "\nbefore internal node insert (6):\n"); + auto padding = std::string(328, '_'); + auto keys = build_key_set({2, 5}, {2, 5}, {2, 5}, padding); + keys.insert(make_ghobj(5, 5, 5, "ns3", "oid3" + std::string(271, '_'), 3, 3)); + keys.insert(make_ghobj(9, 9, 9, "ns~last", "oid~last", 9, 9)); + pool.build_tree(keys).unsafe_get0(); + + logger().info("\n---------------------------------------------" + "\nsplit at stage 0; insert to right front at stage 0\n"); + pool.test_split(make_ghobj(3, 3, 3, "ns3", "oid3" + padding, 2, 3), {1, {1, {1}}}, + {0u, 0u, false, InsertType::BEGIN}).get(); + } + + // Impossible to split at {0, 0, 0} + // Impossible to split at [END, END, END] + }); +} + +struct d_seastore_tm_test_t : + public seastar_test_suite_t, TMTestState { + seastar::future<> set_up_fut() override final { + return tm_setup(); + } + seastar::future<> tear_down_fut() override final { + return tm_teardown(); + } +}; + +TEST_F(d_seastore_tm_test_t, 6_random_insert_leaf_node) +{ + run_async([this] { + constexpr bool TEST_SEASTORE = true; + constexpr bool TRACK_CURSORS = true; + KVPool kvs{{8, 11, 64, 256, 301, 320}, + {8, 16, 128, 512, 576, 640}, + {0, 32}, {0, 10}, {0, 4}}; + auto tree = std::make_unique<TreeBuilder<TRACK_CURSORS>>(kvs, + (TEST_SEASTORE ? NodeExtentManager::create_seastore(*tm) + : NodeExtentManager::create_dummy(IS_DUMMY_SYNC))); + { + auto t = tm->create_transaction(); + tree->bootstrap(*t).unsafe_get(); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + { + auto t = tm->create_transaction(); + tree->insert(*t).unsafe_get(); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + { + auto t = tm->create_transaction(); + tree->get_stats(*t).unsafe_get(); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + if constexpr (TEST_SEASTORE) { + logger().info("seastore replay begin"); + restart(); + tree->reload(NodeExtentManager::create_seastore(*tm)); + logger().info("seastore replay end"); + } + { + // Note: tm->create_weak_transaction() can also work, but too slow. + auto t = tm->create_transaction(); + tree->validate(*t).unsafe_get(); + } + tree.reset(); + }); +} diff --git a/src/test/crimson/seastore/test_block.cc b/src/test/crimson/seastore/test_block.cc new file mode 100644 index 000000000..f3d6531bd --- /dev/null +++ b/src/test/crimson/seastore/test_block.cc @@ -0,0 +1,25 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/crimson/seastore/test_block.h" + +namespace crimson::os::seastore { + + +ceph::bufferlist TestBlock::get_delta() { + ceph::bufferlist bl; + encode(delta, bl); + return bl; +} + + +void TestBlock::apply_delta(const ceph::bufferlist &bl) { + auto biter = bl.begin(); + decltype(delta) deltas; + decode(deltas, biter); + for (auto &&d : deltas) { + set_contents(d.val, d.offset, d.len); + } +} + +} diff --git a/src/test/crimson/seastore/test_block.h b/src/test/crimson/seastore/test_block.h new file mode 100644 index 000000000..44ec65a23 --- /dev/null +++ b/src/test/crimson/seastore/test_block.h @@ -0,0 +1,147 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include <random> + +#include "crimson/os/seastore/transaction_manager.h" + +namespace crimson::os::seastore { + +struct test_extent_desc_t { + size_t len = 0; + unsigned checksum = 0; + + bool operator==(const test_extent_desc_t &rhs) const { + return (len == rhs.len && + checksum == rhs.checksum); + } + bool operator!=(const test_extent_desc_t &rhs) const { + return !(*this == rhs); + } +}; + +struct test_block_delta_t { + int8_t val = 0; + uint16_t offset = 0; + uint16_t len = 0; + + + DENC(test_block_delta_t, v, p) { + DENC_START(1, 1, p); + denc(v.val, p); + denc(v.offset, p); + denc(v.len, p); + DENC_FINISH(p); + } +}; + +inline std::ostream &operator<<( + std::ostream &lhs, const test_extent_desc_t &rhs) { + return lhs << "test_extent_desc_t(len=" << rhs.len + << ", checksum=" << rhs.checksum << ")"; +} + +struct TestBlock : crimson::os::seastore::LogicalCachedExtent { + constexpr static segment_off_t SIZE = 4<<10; + using Ref = TCachedExtentRef<TestBlock>; + + std::vector<test_block_delta_t> delta = {}; + + TestBlock(ceph::bufferptr &&ptr) + : LogicalCachedExtent(std::move(ptr)) {} + TestBlock(const TestBlock &other) + : LogicalCachedExtent(other) {} + + CachedExtentRef duplicate_for_write() final { + return CachedExtentRef(new TestBlock(*this)); + }; + + static constexpr extent_types_t TYPE = extent_types_t::TEST_BLOCK; + extent_types_t get_type() const final { + return TYPE; + } + + ceph::bufferlist get_delta() final; + + void set_contents(char c, uint16_t offset, uint16_t len) { + ::memset(get_bptr().c_str() + offset, c, len); + delta.push_back({c, offset, len}); + } + + void set_contents(char c) { + set_contents(c, 0, get_length()); + } + + test_extent_desc_t get_desc() { + return { get_length(), get_crc32c() }; + } + + void apply_delta(const ceph::bufferlist &bl) final; +}; +using TestBlockRef = TCachedExtentRef<TestBlock>; + +struct TestBlockPhysical : crimson::os::seastore::CachedExtent{ + constexpr static segment_off_t SIZE = 4<<10; + using Ref = TCachedExtentRef<TestBlockPhysical>; + + std::vector<test_block_delta_t> delta = {}; + + TestBlockPhysical(ceph::bufferptr &&ptr) + : CachedExtent(std::move(ptr)) {} + TestBlockPhysical(const TestBlock &other) + : CachedExtent(other) {} + + CachedExtentRef duplicate_for_write() final { + return CachedExtentRef(new TestBlockPhysical(*this)); + }; + + static constexpr extent_types_t TYPE = extent_types_t::TEST_BLOCK_PHYSICAL; + extent_types_t get_type() const final { + return TYPE; + } + + void set_contents(char c, uint16_t offset, uint16_t len) { + ::memset(get_bptr().c_str() + offset, c, len); + } + + void set_contents(char c) { + set_contents(c, 0, get_length()); + } + + ceph::bufferlist get_delta() final { return ceph::bufferlist(); } + + void apply_delta_and_adjust_crc(paddr_t, const ceph::bufferlist &bl) final {} +}; +using TestBlockPhysicalRef = TCachedExtentRef<TestBlockPhysical>; + +struct test_block_mutator_t { + std::uniform_int_distribution<int8_t> + contents_distribution = std::uniform_int_distribution<int8_t>( + std::numeric_limits<int8_t>::min(), + std::numeric_limits<int8_t>::max()); + + std::uniform_int_distribution<uint16_t> + offset_distribution = std::uniform_int_distribution<uint16_t>( + 0, TestBlock::SIZE - 1); + + std::uniform_int_distribution<uint16_t> length_distribution(uint16_t offset) { + return std::uniform_int_distribution<uint16_t>( + 0, TestBlock::SIZE - offset - 1); + } + + + template <typename generator_t> + void mutate(TestBlock &block, generator_t &gen) { + auto offset = offset_distribution(gen); + block.set_contents( + contents_distribution(gen), + offset, + length_distribution(offset)(gen)); + } +}; + +} + +WRITE_CLASS_DENC_BOUNDED(crimson::os::seastore::test_block_delta_t) diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc new file mode 100644 index 000000000..60d5c3497 --- /dev/null +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -0,0 +1,429 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/crimson/gtest_seastar.h" + +#include "crimson/common/log.h" + +#include "crimson/os/seastore/journal.h" +#include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/segment_manager/ephemeral.h" +#include "crimson/os/seastore/lba_manager/btree/btree_lba_manager.h" + +#include "test/crimson/seastore/test_block.h" + +namespace { + [[maybe_unused]] seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_test); + } +} + +using namespace crimson; +using namespace crimson::os; +using namespace crimson::os::seastore; +using namespace crimson::os::seastore::lba_manager; +using namespace crimson::os::seastore::lba_manager::btree; + +struct btree_lba_manager_test : + public seastar_test_suite_t, JournalSegmentProvider { + segment_manager::EphemeralSegmentManagerRef segment_manager; + Journal journal; + Cache cache; + BtreeLBAManagerRef lba_manager; + + const size_t block_size; + + btree_lba_manager_test() + : segment_manager(segment_manager::create_test_ephemeral()), + journal(*segment_manager), + cache(*segment_manager), + lba_manager(new BtreeLBAManager(*segment_manager, cache)), + block_size(segment_manager->get_block_size()) + { + journal.set_segment_provider(this); + } + + segment_id_t next = 0; + get_segment_ret get_segment() final { + return get_segment_ret( + get_segment_ertr::ready_future_marker{}, + next++); + } + + journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; } + void update_journal_tail_committed(journal_seq_t committed) final {} + + auto submit_transaction(TransactionRef t) + { + auto record = cache.try_construct_record(*t); + if (!record) { + ceph_assert(0 == "cannot fail"); + } + + return journal.submit_record(std::move(*record)).safe_then( + [this, t=std::move(t)](auto p) mutable { + auto [addr, seq] = p; + cache.complete_commit(*t, addr, seq); + lba_manager->complete_transaction(*t); + }, + crimson::ct_error::assert_all{}); + } + + seastar::future<> set_up_fut() final { + return segment_manager->init( + ).safe_then([this] { + return journal.open_for_write(); + }).safe_then([this](auto addr) { + return seastar::do_with( + make_transaction(), + [this](auto &transaction) { + cache.init(); + return cache.mkfs(*transaction + ).safe_then([this, &transaction] { + return lba_manager->mkfs(*transaction); + }).safe_then([this, &transaction] { + return submit_transaction(std::move(transaction)); + }); + }); + }).handle_error( + crimson::ct_error::all_same_way([] { + ceph_assert(0 == "error"); + }) + ); + } + + seastar::future<> tear_down_fut() final { + return cache.close( + ).safe_then([this] { + return journal.close(); + }).handle_error( + crimson::ct_error::all_same_way([] { + ASSERT_FALSE("Unable to close"); + }) + ); + } + + + struct test_extent_t { + paddr_t addr; + size_t len = 0; + unsigned refcount = 0; + }; + using test_lba_mapping_t = std::map<laddr_t, test_extent_t>; + test_lba_mapping_t test_lba_mappings; + struct test_transaction_t { + TransactionRef t; + test_lba_mapping_t mappings; + }; + + auto create_transaction() { + auto t = test_transaction_t{ + make_transaction(), + test_lba_mappings + }; + cache.alloc_new_extent<TestBlockPhysical>(*t.t, TestBlockPhysical::SIZE); + return t; + } + + auto create_weak_transaction() { + auto t = test_transaction_t{ + make_weak_transaction(), + test_lba_mappings + }; + return t; + } + + void submit_test_transaction(test_transaction_t t) { + submit_transaction(std::move(t.t)).get0(); + test_lba_mappings.swap(t.mappings); + } + + auto get_overlap(test_transaction_t &t, laddr_t addr, size_t len) { + auto bottom = t.mappings.upper_bound(addr); + if (bottom != t.mappings.begin()) + --bottom; + if (bottom != t.mappings.end() && + bottom->first + bottom->second.len <= addr) + ++bottom; + + auto top = t.mappings.lower_bound(addr + len); + return std::make_pair( + bottom, + top + ); + } + + segment_off_t next_off = 0; + paddr_t get_paddr() { + next_off += block_size; + return make_fake_paddr(next_off); + } + + auto alloc_mapping( + test_transaction_t &t, + laddr_t hint, + size_t len, + paddr_t paddr) { + auto ret = lba_manager->alloc_extent(*t.t, hint, len, paddr).unsafe_get0(); + logger().debug("alloc'd: {}", *ret); + EXPECT_EQ(len, ret->get_length()); + auto [b, e] = get_overlap(t, ret->get_laddr(), len); + EXPECT_EQ(b, e); + t.mappings.emplace( + std::make_pair( + ret->get_laddr(), + test_extent_t{ + ret->get_paddr(), + ret->get_length(), + 1 + } + )); + return ret; + } + + auto set_mapping( + test_transaction_t &t, + laddr_t addr, + size_t len, + paddr_t paddr) { + auto [b, e] = get_overlap(t, addr, len); + EXPECT_EQ(b, e); + + auto ret = lba_manager->set_extent(*t.t, addr, len, paddr).unsafe_get0(); + EXPECT_EQ(addr, ret->get_laddr()); + EXPECT_EQ(len, ret->get_length()); + EXPECT_EQ(paddr, ret->get_paddr()); + t.mappings.emplace( + std::make_pair( + ret->get_laddr(), + test_extent_t{ + ret->get_paddr(), + ret->get_length(), + 1 + } + )); + return ret; + } + + auto decref_mapping( + test_transaction_t &t, + laddr_t addr) { + return decref_mapping(t, t.mappings.find(addr)); + } + + void decref_mapping( + test_transaction_t &t, + test_lba_mapping_t::iterator target) { + ceph_assert(target != t.mappings.end()); + ceph_assert(target->second.refcount > 0); + target->second.refcount--; + + auto refcnt = lba_manager->decref_extent( + *t.t, + target->first).unsafe_get0().refcount; + EXPECT_EQ(refcnt, target->second.refcount); + if (target->second.refcount == 0) { + t.mappings.erase(target); + } + } + + auto incref_mapping( + test_transaction_t &t, + laddr_t addr) { + return incref_mapping(t, t.mappings.find(addr)); + } + + void incref_mapping( + test_transaction_t &t, + test_lba_mapping_t::iterator target) { + ceph_assert(target->second.refcount > 0); + target->second.refcount++; + auto refcnt = lba_manager->incref_extent( + *t.t, + target->first).unsafe_get0().refcount; + EXPECT_EQ(refcnt, target->second.refcount); + } + + std::vector<laddr_t> get_mapped_addresses() { + std::vector<laddr_t> addresses; + addresses.reserve(test_lba_mappings.size()); + for (auto &i: test_lba_mappings) { + addresses.push_back(i.first); + } + return addresses; + } + + std::vector<laddr_t> get_mapped_addresses(test_transaction_t &t) { + std::vector<laddr_t> addresses; + addresses.reserve(t.mappings.size()); + for (auto &i: t.mappings) { + addresses.push_back(i.first); + } + return addresses; + } + + void check_mappings() { + auto t = create_transaction(); + check_mappings(t); + } + + void check_mappings(test_transaction_t &t) { + for (auto &&i: t.mappings) { + auto ret_list = lba_manager->get_mapping( + *t.t, i.first, i.second.len + ).unsafe_get0(); + EXPECT_EQ(ret_list.size(), 1); + auto &ret = *ret_list.begin(); + EXPECT_EQ(i.second.addr, ret->get_paddr()); + EXPECT_EQ(i.first, ret->get_laddr()); + EXPECT_EQ(i.second.len, ret->get_length()); + } + lba_manager->scan_mappings( + *t.t, + 0, + L_ADDR_MAX, + [iter=t.mappings.begin(), &t](auto l, auto p, auto len) mutable { + EXPECT_NE(iter, t.mappings.end()); + EXPECT_EQ(l, iter->first); + EXPECT_EQ(p, iter->second.addr); + EXPECT_EQ(len, iter->second.len); + ++iter; + }).unsafe_get(); + } +}; + +TEST_F(btree_lba_manager_test, basic) +{ + run_async([this] { + laddr_t laddr = 0x12345678 * block_size; + { + // write initial mapping + auto t = create_transaction(); + check_mappings(t); // check in progress transaction sees mapping + check_mappings(); // check concurrent does not + auto ret = alloc_mapping(t, laddr, block_size, get_paddr()); + submit_test_transaction(std::move(t)); + } + check_mappings(); // check new transaction post commit sees it + }); +} + +TEST_F(btree_lba_manager_test, force_split) +{ + run_async([this] { + for (unsigned i = 0; i < 40; ++i) { + auto t = create_transaction(); + logger().debug("opened transaction"); + for (unsigned j = 0; j < 5; ++j) { + auto ret = alloc_mapping(t, 0, block_size, get_paddr()); + if ((i % 10 == 0) && (j == 3)) { + check_mappings(t); + check_mappings(); + } + } + logger().debug("submitting transaction"); + submit_test_transaction(std::move(t)); + check_mappings(); + } + }); +} + +TEST_F(btree_lba_manager_test, force_split_merge) +{ + run_async([this] { + for (unsigned i = 0; i < 80; ++i) { + auto t = create_transaction(); + logger().debug("opened transaction"); + for (unsigned j = 0; j < 5; ++j) { + auto ret = alloc_mapping(t, 0, block_size, get_paddr()); + // just to speed things up a bit + if ((i % 100 == 0) && (j == 3)) { + check_mappings(t); + check_mappings(); + } + incref_mapping(t, ret->get_laddr()); + decref_mapping(t, ret->get_laddr()); + } + logger().debug("submitting transaction"); + submit_test_transaction(std::move(t)); + if (i % 50 == 0) { + check_mappings(); + } + } + { + auto addresses = get_mapped_addresses(); + auto t = create_transaction(); + for (unsigned i = 0; i != addresses.size(); ++i) { + if (i % 2 == 0) { + incref_mapping(t, addresses[i]); + decref_mapping(t, addresses[i]); + decref_mapping(t, addresses[i]); + } + logger().debug("submitting transaction"); + if (i % 7 == 0) { + submit_test_transaction(std::move(t)); + t = create_transaction(); + } + if (i % 13 == 0) { + check_mappings(); + check_mappings(t); + } + } + submit_test_transaction(std::move(t)); + } + { + auto addresses = get_mapped_addresses(); + auto t = create_transaction(); + for (unsigned i = 0; i != addresses.size(); ++i) { + incref_mapping(t, addresses[i]); + decref_mapping(t, addresses[i]); + decref_mapping(t, addresses[i]); + } + check_mappings(t); + submit_test_transaction(std::move(t)); + check_mappings(); + } + }); +} + +TEST_F(btree_lba_manager_test, single_transaction_split_merge) +{ + run_async([this] { + { + auto t = create_transaction(); + for (unsigned i = 0; i < 600; ++i) { + alloc_mapping(t, 0, block_size, get_paddr()); + } + check_mappings(t); + submit_test_transaction(std::move(t)); + } + check_mappings(); + + { + auto addresses = get_mapped_addresses(); + auto t = create_transaction(); + for (unsigned i = 0; i != addresses.size(); ++i) { + if (i % 4 != 0) { + decref_mapping(t, addresses[i]); + } + } + check_mappings(t); + submit_test_transaction(std::move(t)); + } + check_mappings(); + + { + auto t = create_transaction(); + for (unsigned i = 0; i < 600; ++i) { + alloc_mapping(t, 0, block_size, get_paddr()); + } + auto addresses = get_mapped_addresses(t); + for (unsigned i = 0; i != addresses.size(); ++i) { + decref_mapping(t, addresses[i]); + } + check_mappings(t); + submit_test_transaction(std::move(t)); + } + check_mappings(); + }); +} diff --git a/src/test/crimson/seastore/test_extmap_manager.cc b/src/test/crimson/seastore/test_extmap_manager.cc new file mode 100644 index 000000000..8b2588011 --- /dev/null +++ b/src/test/crimson/seastore/test_extmap_manager.cc @@ -0,0 +1,283 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/crimson/gtest_seastar.h" +#include "test/crimson/seastore/transaction_manager_test_state.h" + +#include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/transaction_manager.h" +#include "crimson/os/seastore/segment_manager.h" +#include "crimson/os/seastore/extentmap_manager.h" + +#include "test/crimson/seastore/test_block.h" + +using namespace crimson; +using namespace crimson::os; +using namespace crimson::os::seastore; + +namespace { + [[maybe_unused]] seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_test); + } +} + + +struct extentmap_manager_test_t : + public seastar_test_suite_t, + TMTestState { + + ExtentMapManagerRef extmap_manager; + + extentmap_manager_test_t() {} + + seastar::future<> set_up_fut() final { + return tm_setup().then([this] { + extmap_manager = extentmap_manager::create_extentmap_manager(*tm); + return seastar::now(); + }); + } + + seastar::future<> tear_down_fut() final { + return tm_teardown().then([this] { + extmap_manager.reset(); + return seastar::now(); + }); + } + + using test_extmap_t = std::map<uint32_t, lext_map_val_t>; + test_extmap_t test_ext_mappings; + + extent_mapping_t insert_extent( + extmap_root_t &extmap_root, + Transaction &t, + uint32_t lo, + lext_map_val_t val) { + auto extent = extmap_manager->add_lextent(extmap_root, t, lo, val).unsafe_get0(); + EXPECT_EQ(lo, extent.logical_offset); + EXPECT_EQ(val.laddr, extent.laddr); + EXPECT_EQ(val.length, extent.length); + test_ext_mappings.emplace(extent.logical_offset, + lext_map_val_t{extent.laddr, extent.length}); + return extent; + } + + extent_map_list_t find_extent( + extmap_root_t &extmap_root, + Transaction &t, + uint32_t lo, + uint32_t len) { + auto extent = extmap_manager->find_lextent(extmap_root, t, lo, len).unsafe_get0(); + EXPECT_EQ(lo, extent.front().logical_offset); + EXPECT_EQ(len, extent.front().length); + return extent; + } + + extent_map_list_t findno_extent( + extmap_root_t &extmap_root, + Transaction &t, + uint32_t lo, + uint32_t len) { + auto extent = extmap_manager->find_lextent(extmap_root, t, lo, len).unsafe_get0(); + EXPECT_EQ(extent.empty(), true); + return extent; + } + + void rm_extent( + extmap_root_t &extmap_root, + Transaction &t, + uint32_t lo, + lext_map_val_t val ) { + auto ret = extmap_manager->rm_lextent(extmap_root, t, lo, val).unsafe_get0(); + EXPECT_TRUE(ret); + test_ext_mappings.erase(lo); + } + + void check_mappings(extmap_root_t &extmap_root, Transaction &t) { + for (const auto& [lo, ext]: test_ext_mappings){ + const auto ext_list = find_extent(extmap_root, t, lo, ext.length); + ASSERT_EQ(ext_list.size(), 1); + const auto& ext_map = ext_list.front(); + EXPECT_EQ(ext.laddr, ext_map.laddr); + EXPECT_EQ(ext.length, ext_map.length); + } + } + + void check_mappings(extmap_root_t &extmap_root) { + auto t = tm->create_transaction(); + check_mappings(extmap_root, *t); + } + + void replay() { + logger().debug("{}: begin", __func__); + restart(); + extmap_manager = extentmap_manager::create_extentmap_manager(*tm); + logger().debug("{}: end", __func__); + } + + +}; + +TEST_F(extentmap_manager_test_t, basic) +{ + run_async([this] { + extmap_root_t extmap_root(0, L_ADDR_NULL); + { + auto t = tm->create_transaction(); + extmap_root = extmap_manager->initialize_extmap(*t).unsafe_get0(); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + + uint32_t len = 4096; + uint32_t lo = 0x1 * len; + { + auto t = tm->create_transaction(); + logger().debug("first transaction"); + [[maybe_unused]] auto addref = insert_extent(extmap_root, *t, lo, {lo, len}); + [[maybe_unused]] auto seekref = find_extent(extmap_root, *t, lo, len); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + { + auto t = tm->create_transaction(); + logger().debug("second transaction"); + auto seekref = find_extent(extmap_root, *t, lo, len); + rm_extent(extmap_root, *t, lo, {seekref.front().laddr, len}); + [[maybe_unused]] auto seekref2 = findno_extent(extmap_root, *t, lo, len); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + { + auto t = tm->create_transaction(); + logger().debug("third transaction"); + [[maybe_unused]] auto seekref = findno_extent(extmap_root, *t, lo, len); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + }); +} + +TEST_F(extentmap_manager_test_t, force_leafnode_split) +{ + run_async([this] { + extmap_root_t extmap_root(0, L_ADDR_NULL); + { + auto t = tm->create_transaction(); + extmap_root = extmap_manager->initialize_extmap(*t).unsafe_get0(); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + uint32_t len = 4096; + uint32_t lo = 0; + for (unsigned i = 0; i < 40; i++) { + auto t = tm->create_transaction(); + logger().debug("opened transaction"); + for (unsigned j = 0; j < 10; ++j) { + [[maybe_unused]] auto addref = insert_extent(extmap_root, *t, lo, {lo, len}); + lo += len; + if ((i % 20 == 0) && (j == 5)) { + check_mappings(extmap_root, *t); + } + } + logger().debug("force split submit transaction i = {}", i); + tm->submit_transaction(std::move(t)).unsafe_get(); + check_mappings(extmap_root); + } + }); + +} + +TEST_F(extentmap_manager_test_t, force_leafnode_split_merge) +{ + run_async([this] { + extmap_root_t extmap_root(0, L_ADDR_NULL); + { + auto t = tm->create_transaction(); + extmap_root = extmap_manager->initialize_extmap(*t).unsafe_get0(); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + uint32_t len = 4096; + uint32_t lo = 0; + for (unsigned i = 0; i < 80; i++) { + auto t = tm->create_transaction(); + logger().debug("opened split_merge transaction"); + for (unsigned j = 0; j < 5; ++j) { + [[maybe_unused]] auto addref = insert_extent(extmap_root, *t, lo, {lo, len}); + lo += len; + if ((i % 10 == 0) && (j == 3)) { + check_mappings(extmap_root, *t); + } + } + logger().debug("submitting transaction"); + tm->submit_transaction(std::move(t)).unsafe_get(); + if (i % 50 == 0) { + check_mappings(extmap_root); + } + } + auto t = tm->create_transaction(); + int i = 0; + for (auto iter = test_ext_mappings.begin(); iter != test_ext_mappings.end();) { + auto [lo, ext] = *iter; + ++iter; + if (i % 3 != 0) { + rm_extent(extmap_root, *t, lo, ext); + } + i++; + + if (i % 10 == 0) { + logger().debug("submitting transaction i= {}", i); + tm->submit_transaction(std::move(t)).unsafe_get(); + t = tm->create_transaction(); + } + if (i % 100 == 0) { + logger().debug("check_mappings i= {}", i); + check_mappings(extmap_root, *t); + check_mappings(extmap_root); + } + } + logger().debug("finally submitting transaction "); + tm->submit_transaction(std::move(t)).unsafe_get(); + }); +} + +TEST_F(extentmap_manager_test_t, force_leafnode_split_merge_replay) +{ + run_async([this] { + extmap_root_t extmap_root(0, L_ADDR_NULL); + { + auto t = tm->create_transaction(); + extmap_root = extmap_manager->initialize_extmap(*t).unsafe_get0(); + tm->submit_transaction(std::move(t)).unsafe_get(); + replay(); + } + uint32_t len = 4096; + uint32_t lo = 0; + for (unsigned i = 0; i < 50; i++) { + auto t = tm->create_transaction(); + logger().debug("opened split_merge transaction"); + for (unsigned j = 0; j < 5; ++j) { + [[maybe_unused]] auto addref = insert_extent(extmap_root, *t, lo, {lo, len}); + lo += len; + } + logger().debug("submitting transaction"); + tm->submit_transaction(std::move(t)).unsafe_get(); + } + replay(); + auto t = tm->create_transaction(); + int i = 0; + for (auto iter = test_ext_mappings.begin(); iter != test_ext_mappings.end();) { + auto [lo, ext] = *iter; + ++iter; + rm_extent(extmap_root, *t, lo, ext); + i++; + + if (i % 10 == 0) { + logger().debug("submitting transaction i= {}", i); + tm->submit_transaction(std::move(t)).unsafe_get(); + t = tm->create_transaction(); + } + if (i% 100 == 0){ + check_mappings(extmap_root); + } + } + logger().debug("finally submitting transaction "); + tm->submit_transaction(std::move(t)).unsafe_get(); + replay(); + check_mappings(extmap_root); + }); +} diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc new file mode 100644 index 000000000..913668b08 --- /dev/null +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/crimson/gtest_seastar.h" + +#include "crimson/common/log.h" +#include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/segment_manager/ephemeral.h" + +#include "test/crimson/seastore/test_block.h" + +using namespace crimson; +using namespace crimson::os; +using namespace crimson::os::seastore; + +namespace { + [[maybe_unused]] seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_test); + } +} + +struct cache_test_t : public seastar_test_suite_t { + segment_manager::EphemeralSegmentManagerRef segment_manager; + Cache cache; + paddr_t current{0, 0}; + journal_seq_t seq; + + cache_test_t() + : segment_manager(segment_manager::create_test_ephemeral()), + cache(*segment_manager) {} + + seastar::future<std::optional<paddr_t>> submit_transaction( + TransactionRef t) { + auto record = cache.try_construct_record(*t); + if (!record) { + return seastar::make_ready_future<std::optional<paddr_t>>( + std::nullopt); + } + + bufferlist bl; + for (auto &&block : record->extents) { + bl.append(block.bl); + } + + ceph_assert((segment_off_t)bl.length() < + segment_manager->get_segment_size()); + if (current.offset + (segment_off_t)bl.length() > + segment_manager->get_segment_size()) + current = paddr_t{current.segment + 1, 0}; + + auto prev = current; + current.offset += bl.length(); + return segment_manager->segment_write( + prev, + std::move(bl), + true + ).safe_then( + [this, prev, t=std::move(t)]() mutable { + cache.complete_commit(*t, prev, seq /* TODO */); + return seastar::make_ready_future<std::optional<paddr_t>>(prev); + }, + crimson::ct_error::all_same_way([](auto e) { + ASSERT_FALSE("failed to submit"); + }) + ); + } + + auto get_transaction() { + return make_transaction(); + } + + seastar::future<> set_up_fut() final { + return segment_manager->init( + ).safe_then( + [this] { + return seastar::do_with( + make_transaction(), + [this](auto &transaction) { + cache.init(); + return cache.mkfs(*transaction).safe_then( + [this, &transaction] { + return submit_transaction(std::move(transaction)).then( + [](auto p) { + ASSERT_TRUE(p); + }); + }); + }); + }).handle_error( + crimson::ct_error::all_same_way([](auto e) { + ASSERT_FALSE("failed to submit"); + }) + ); + } + + seastar::future<> tear_down_fut() final { + return cache.close().handle_error( + Cache::close_ertr::assert_all{}); + } +}; + +TEST_F(cache_test_t, test_addr_fixup) +{ + run_async([this] { + paddr_t addr; + int csum = 0; + { + auto t = get_transaction(); + auto extent = cache.alloc_new_extent<TestBlockPhysical>( + *t, + TestBlockPhysical::SIZE); + extent->set_contents('c'); + csum = extent->get_crc32c(); + auto ret = submit_transaction(std::move(t)).get0(); + ASSERT_TRUE(ret); + addr = extent->get_paddr(); + } + { + auto t = get_transaction(); + auto extent = cache.get_extent<TestBlockPhysical>( + *t, + addr, + TestBlockPhysical::SIZE).unsafe_get0(); + ASSERT_EQ(extent->get_paddr(), addr); + ASSERT_EQ(extent->get_crc32c(), csum); + } + }); +} + +TEST_F(cache_test_t, test_dirty_extent) +{ + run_async([this] { + paddr_t addr; + int csum = 0; + int csum2 = 0; + { + // write out initial test block + auto t = get_transaction(); + auto extent = cache.alloc_new_extent<TestBlockPhysical>( + *t, + TestBlockPhysical::SIZE); + extent->set_contents('c'); + csum = extent->get_crc32c(); + auto reladdr = extent->get_paddr(); + ASSERT_TRUE(reladdr.is_relative()); + { + // test that read with same transaction sees new block though + // uncommitted + auto extent = cache.get_extent<TestBlockPhysical>( + *t, + reladdr, + TestBlockPhysical::SIZE).unsafe_get0(); + ASSERT_TRUE(extent->is_clean()); + ASSERT_TRUE(extent->is_pending()); + ASSERT_TRUE(extent->get_paddr().is_relative()); + ASSERT_EQ(extent->get_version(), 0); + ASSERT_EQ(csum, extent->get_crc32c()); + } + auto ret = submit_transaction(std::move(t)).get0(); + ASSERT_TRUE(ret); + addr = extent->get_paddr(); + } + { + // test that consecutive reads on the same extent get the same ref + auto t = get_transaction(); + auto extent = cache.get_extent<TestBlockPhysical>( + *t, + addr, + TestBlockPhysical::SIZE).unsafe_get0(); + auto t2 = get_transaction(); + auto extent2 = cache.get_extent<TestBlockPhysical>( + *t2, + addr, + TestBlockPhysical::SIZE).unsafe_get0(); + ASSERT_EQ(&*extent, &*extent2); + } + { + // read back test block + auto t = get_transaction(); + auto extent = cache.get_extent<TestBlockPhysical>( + *t, + addr, + TestBlockPhysical::SIZE).unsafe_get0(); + // duplicate and reset contents + extent = cache.duplicate_for_write(*t, extent)->cast<TestBlockPhysical>(); + extent->set_contents('c'); + csum2 = extent->get_crc32c(); + ASSERT_EQ(extent->get_paddr(), addr); + { + // test that concurrent read with fresh transaction sees old + // block + auto t2 = get_transaction(); + auto extent = cache.get_extent<TestBlockPhysical>( + *t2, + addr, + TestBlockPhysical::SIZE).unsafe_get0(); + ASSERT_TRUE(extent->is_clean()); + ASSERT_FALSE(extent->is_pending()); + ASSERT_EQ(addr, extent->get_paddr()); + ASSERT_EQ(extent->get_version(), 0); + ASSERT_EQ(csum, extent->get_crc32c()); + } + { + // test that read with same transaction sees new block + auto extent = cache.get_extent<TestBlockPhysical>( + *t, + addr, + TestBlockPhysical::SIZE).unsafe_get0(); + ASSERT_TRUE(extent->is_dirty()); + ASSERT_TRUE(extent->is_pending()); + ASSERT_EQ(addr, extent->get_paddr()); + ASSERT_EQ(extent->get_version(), 1); + ASSERT_EQ(csum2, extent->get_crc32c()); + } + // submit transaction + auto ret = submit_transaction(std::move(t)).get0(); + ASSERT_TRUE(ret); + ASSERT_TRUE(extent->is_dirty()); + ASSERT_EQ(addr, extent->get_paddr()); + ASSERT_EQ(extent->get_version(), 1); + ASSERT_EQ(extent->get_crc32c(), csum2); + } + { + // test that fresh transaction now sees newly dirty block + auto t = get_transaction(); + auto extent = cache.get_extent<TestBlockPhysical>( + *t, + addr, + TestBlockPhysical::SIZE).unsafe_get0(); + ASSERT_TRUE(extent->is_dirty()); + ASSERT_EQ(addr, extent->get_paddr()); + ASSERT_EQ(extent->get_version(), 1); + ASSERT_EQ(csum2, extent->get_crc32c()); + } + }); +} diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc new file mode 100644 index 000000000..0bed505ff --- /dev/null +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -0,0 +1,260 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/crimson/gtest_seastar.h" + +#include <random> + +#include "crimson/common/log.h" +#include "crimson/os/seastore/journal.h" +#include "crimson/os/seastore/segment_manager/ephemeral.h" + +using namespace crimson; +using namespace crimson::os; +using namespace crimson::os::seastore; + +namespace { + [[maybe_unused]] seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_test); + } +} + +struct record_validator_t { + record_t record; + paddr_t record_final_offset; + + template <typename... T> + record_validator_t(T&&... record) : record(std::forward<T>(record)...) {} + + void validate(SegmentManager &manager) { + paddr_t addr = make_record_relative_paddr(0); + for (auto &&block : record.extents) { + auto test = manager.read( + record_final_offset.add_relative(addr), + block.bl.length()).unsafe_get0(); + addr.offset += block.bl.length(); + bufferlist bl; + bl.push_back(test); + ASSERT_EQ( + bl.length(), + block.bl.length()); + ASSERT_EQ( + bl.begin().crc32c(bl.length(), 1), + block.bl.begin().crc32c(block.bl.length(), 1)); + } + } + + auto get_replay_handler() { + auto checker = [this, iter=record.deltas.begin()] ( + paddr_t base, + const delta_info_t &di) mutable { + EXPECT_EQ(base, record_final_offset); + ceph_assert(iter != record.deltas.end()); + EXPECT_EQ(di, *iter++); + EXPECT_EQ(base, record_final_offset); + return iter != record.deltas.end(); + }; + if (record.deltas.size()) { + return std::make_optional(std::move(checker)); + } else { + return std::optional<decltype(checker)>(); + } + } +}; + +struct journal_test_t : seastar_test_suite_t, JournalSegmentProvider { + segment_manager::EphemeralSegmentManagerRef segment_manager; + std::unique_ptr<Journal> journal; + + std::vector<record_validator_t> records; + + std::default_random_engine generator; + + const segment_off_t block_size; + + journal_test_t() + : segment_manager(segment_manager::create_test_ephemeral()), + block_size(segment_manager->get_block_size()) + { + } + + segment_id_t next = 0; + get_segment_ret get_segment() final { + return get_segment_ret( + get_segment_ertr::ready_future_marker{}, + next++); + } + + journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; } + void update_journal_tail_committed(journal_seq_t paddr) final {} + + seastar::future<> set_up_fut() final { + journal.reset(new Journal(*segment_manager)); + journal->set_segment_provider(this); + return segment_manager->init( + ).safe_then([this] { + return journal->open_for_write(); + }).safe_then( + [](auto){}, + crimson::ct_error::all_same_way([] { + ASSERT_FALSE("Unable to mount"); + })); + } + + template <typename T> + auto replay(T &&f) { + return journal->close( + ).safe_then([this, f=std::move(f)]() mutable { + journal.reset(new Journal(*segment_manager)); + journal->set_segment_provider(this); + return journal->replay(std::forward<T>(std::move(f))); + }).safe_then([this] { + return journal->open_for_write(); + }); + } + + auto replay_and_check() { + auto record_iter = records.begin(); + decltype(record_iter->get_replay_handler()) delta_checker = std::nullopt; + auto advance = [this, &record_iter, &delta_checker] { + ceph_assert(!delta_checker); + while (record_iter != records.end()) { + auto checker = record_iter->get_replay_handler(); + record_iter++; + if (checker) { + delta_checker.emplace(std::move(*checker)); + break; + } + } + }; + advance(); + replay( + [&advance, + &delta_checker] + (auto seq, auto base, const auto &di) mutable { + if (!delta_checker) { + EXPECT_FALSE("No Deltas Left"); + } + if (!(*delta_checker)(base, di)) { + delta_checker = std::nullopt; + advance(); + } + return Journal::replay_ertr::now(); + }).unsafe_get0(); + ASSERT_EQ(record_iter, records.end()); + for (auto &i : records) { + i.validate(*segment_manager); + } + } + + template <typename... T> + auto submit_record(T&&... _record) { + auto record{std::forward<T>(_record)...}; + records.push_back(record); + auto [addr, _] = journal->submit_record(std::move(record)).unsafe_get0(); + records.back().record_final_offset = addr; + return addr; + } + + seastar::future<> tear_down_fut() final { + return seastar::now(); + } + + extent_t generate_extent(size_t blocks) { + std::uniform_int_distribution<char> distribution( + std::numeric_limits<char>::min(), + std::numeric_limits<char>::max() + ); + char contents = distribution(generator); + bufferlist bl; + bl.append(buffer::ptr(buffer::create(blocks * block_size, contents))); + return extent_t{extent_types_t::TEST_BLOCK, L_ADDR_NULL, bl}; + } + + delta_info_t generate_delta(size_t bytes) { + std::uniform_int_distribution<char> distribution( + std::numeric_limits<char>::min(), + std::numeric_limits<char>::max() + ); + char contents = distribution(generator); + bufferlist bl; + bl.append(buffer::ptr(buffer::create(bytes, contents))); + return delta_info_t{ + extent_types_t::TEST_BLOCK, + paddr_t{}, + L_ADDR_NULL, + 0, 0, + block_size, + 1, + bl + }; + } +}; + +TEST_F(journal_test_t, replay_one_journal_segment) +{ + run_async([this] { + submit_record(record_t{ + { generate_extent(1), generate_extent(2) }, + { generate_delta(23), generate_delta(30) } + }); + replay_and_check(); + }); +} + +TEST_F(journal_test_t, replay_two_records) +{ + run_async([this] { + submit_record(record_t{ + { generate_extent(1), generate_extent(2) }, + { generate_delta(23), generate_delta(30) } + }); + submit_record(record_t{ + { generate_extent(4), generate_extent(1) }, + { generate_delta(23), generate_delta(400) } + }); + replay_and_check(); + }); +} + +TEST_F(journal_test_t, replay_twice) +{ + run_async([this] { + submit_record(record_t{ + { generate_extent(1), generate_extent(2) }, + { generate_delta(23), generate_delta(30) } + }); + submit_record(record_t{ + { generate_extent(4), generate_extent(1) }, + { generate_delta(23), generate_delta(400) } + }); + replay_and_check(); + submit_record(record_t{ + { generate_extent(2), generate_extent(5) }, + { generate_delta(230), generate_delta(40) } + }); + replay_and_check(); + }); +} + +TEST_F(journal_test_t, roll_journal_and_replay) +{ + run_async([this] { + paddr_t current = submit_record( + record_t{ + { generate_extent(1), generate_extent(2) }, + { generate_delta(23), generate_delta(30) } + }); + auto starting_segment = current.segment; + unsigned so_far = 0; + while (current.segment == starting_segment) { + current = submit_record(record_t{ + { generate_extent(512), generate_extent(512) }, + { generate_delta(23), generate_delta(400) } + }); + ++so_far; + ASSERT_FALSE(so_far > 10); + } + replay_and_check(); + }); +} diff --git a/src/test/crimson/seastore/test_transaction_manager.cc b/src/test/crimson/seastore/test_transaction_manager.cc new file mode 100644 index 000000000..9906f938a --- /dev/null +++ b/src/test/crimson/seastore/test_transaction_manager.cc @@ -0,0 +1,495 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include <random> + +#include "test/crimson/gtest_seastar.h" +#include "test/crimson/seastore/transaction_manager_test_state.h" + +#include "crimson/os/seastore/segment_cleaner.h" +#include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/transaction_manager.h" +#include "crimson/os/seastore/segment_manager/ephemeral.h" +#include "crimson/os/seastore/segment_manager.h" + +#include "test/crimson/seastore/test_block.h" + +using namespace crimson; +using namespace crimson::os; +using namespace crimson::os::seastore; + +namespace { + [[maybe_unused]] seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_test); + } +} + +struct test_extent_record_t { + test_extent_desc_t desc; + unsigned refcount = 0; + test_extent_record_t() = default; + test_extent_record_t( + const test_extent_desc_t &desc, + unsigned refcount) : desc(desc), refcount(refcount) {} + + void update(const test_extent_desc_t &to) { + desc = to; + } + + bool operator==(const test_extent_desc_t &rhs) const { + return desc == rhs; + } + bool operator!=(const test_extent_desc_t &rhs) const { + return desc != rhs; + } +}; + +std::ostream &operator<<(std::ostream &lhs, const test_extent_record_t &rhs) { + return lhs << "test_extent_record_t(" << rhs.desc + << ", refcount=" << rhs.refcount << ")"; +} + +struct transaction_manager_test_t : + public seastar_test_suite_t, + TMTestState { + + std::random_device rd; + std::mt19937 gen; + + transaction_manager_test_t() + : gen(rd()) { + init(); + } + + laddr_t get_random_laddr(size_t block_size, laddr_t limit) { + return block_size * + std::uniform_int_distribution<>(0, (limit / block_size) - 1)(gen); + } + + char get_random_contents() { + return static_cast<char>(std::uniform_int_distribution<>(0, 255)(gen)); + } + + seastar::future<> set_up_fut() final { + return tm_setup(); + } + + seastar::future<> tear_down_fut() final { + return tm_teardown(); + } + + struct test_extents_t : std::map<laddr_t, test_extent_record_t> { + private: + void check_available(laddr_t addr, extent_len_t len) { + auto iter = upper_bound(addr); + if (iter != begin()) { + auto liter = iter; + liter--; + EXPECT_FALSE(liter->first + liter->second.desc.len > addr); + } + if (iter != end()) { + EXPECT_FALSE(iter->first < addr + len); + } + } + void check_hint(laddr_t hint, laddr_t addr, extent_len_t len) { + auto iter = lower_bound(hint); + laddr_t last = hint; + while (true) { + if (iter == end() || iter->first > addr) { + EXPECT_EQ(addr, last); + break; + } + EXPECT_FALSE(iter->first - last > len); + last = iter->first + iter->second.desc.len; + ++iter; + } + } + public: + void insert(TestBlock &extent) { + check_available(extent.get_laddr(), extent.get_length()); + emplace( + std::make_pair( + extent.get_laddr(), + test_extent_record_t{extent.get_desc(), 1} + )); + } + void alloced(laddr_t hint, TestBlock &extent) { + check_hint(hint, extent.get_laddr(), extent.get_length()); + insert(extent); + } + } test_mappings; + + struct test_transaction_t { + TransactionRef t; + test_extents_t mappings; + }; + + test_transaction_t create_transaction() { + return { tm->create_transaction(), test_mappings }; + } + + test_transaction_t create_weak_transaction() { + return { tm->create_weak_transaction(), test_mappings }; + } + + TestBlockRef alloc_extent( + test_transaction_t &t, + laddr_t hint, + extent_len_t len, + char contents) { + auto extent = tm->alloc_extent<TestBlock>( + *(t.t), + hint, + len).unsafe_get0(); + extent->set_contents(contents); + EXPECT_FALSE(t.mappings.count(extent->get_laddr())); + EXPECT_EQ(len, extent->get_length()); + t.mappings.alloced(hint, *extent); + return extent; + } + + TestBlockRef alloc_extent( + test_transaction_t &t, + laddr_t hint, + extent_len_t len) { + return alloc_extent( + t, + hint, + len, + get_random_contents()); + } + + bool check_usage() { + auto t = create_weak_transaction(); + SpaceTrackerIRef tracker(segment_cleaner->get_empty_space_tracker()); + lba_manager->scan_mapped_space( + *t.t, + [&tracker](auto offset, auto len) { + tracker->allocate( + offset.segment, + offset.offset, + len); + }).unsafe_get0(); + return segment_cleaner->debug_check_space(*tracker); + } + + void replay() { + logger().debug("{}: begin", __func__); + EXPECT_TRUE(check_usage()); + restart(); + logger().debug("{}: end", __func__); + } + + void check() { + check_mappings(); + check_usage(); + } + + void check_mappings() { + auto t = create_weak_transaction(); + check_mappings(t); + } + + TestBlockRef get_extent( + test_transaction_t &t, + laddr_t addr, + extent_len_t len) { + ceph_assert(t.mappings.count(addr)); + ceph_assert(t.mappings[addr].desc.len == len); + + auto ret_list = tm->read_extents<TestBlock>( + *t.t, addr, len + ).unsafe_get0(); + EXPECT_EQ(ret_list.size(), 1); + auto &ext = ret_list.begin()->second; + auto &laddr = ret_list.begin()->first; + EXPECT_EQ(addr, laddr); + EXPECT_EQ(addr, ext->get_laddr()); + return ext; + } + + test_block_mutator_t mutator; + TestBlockRef mutate_extent( + test_transaction_t &t, + TestBlockRef ref) { + ceph_assert(t.mappings.count(ref->get_laddr())); + ceph_assert(t.mappings[ref->get_laddr()].desc.len == ref->get_length()); + auto ext = tm->get_mutable_extent(*t.t, ref)->cast<TestBlock>(); + EXPECT_EQ(ext->get_laddr(), ref->get_laddr()); + EXPECT_EQ(ext->get_desc(), ref->get_desc()); + mutator.mutate(*ext, gen); + t.mappings[ext->get_laddr()].update(ext->get_desc()); + return ext; + } + + void inc_ref(test_transaction_t &t, laddr_t offset) { + ceph_assert(t.mappings.count(offset)); + ceph_assert(t.mappings[offset].refcount > 0); + auto refcnt = tm->inc_ref(*t.t, offset).unsafe_get0(); + t.mappings[offset].refcount++; + EXPECT_EQ(refcnt, t.mappings[offset].refcount); + } + + void dec_ref(test_transaction_t &t, laddr_t offset) { + ceph_assert(t.mappings.count(offset)); + ceph_assert(t.mappings[offset].refcount > 0); + auto refcnt = tm->dec_ref(*t.t, offset).unsafe_get0(); + t.mappings[offset].refcount--; + EXPECT_EQ(refcnt, t.mappings[offset].refcount); + if (t.mappings[offset].refcount == 0) { + t.mappings.erase(offset); + } + } + + void check_mappings(test_transaction_t &t) { + for (auto &i: t.mappings) { + logger().debug("check_mappings: {}->{}", i.first, i.second); + auto ext = get_extent(t, i.first, i.second.desc.len); + EXPECT_EQ(i.second, ext->get_desc()); + } + auto lt = create_weak_transaction(); + lba_manager->scan_mappings( + *lt.t, + 0, + L_ADDR_MAX, + [iter=lt.mappings.begin(), <](auto l, auto p, auto len) mutable { + EXPECT_NE(iter, lt.mappings.end()); + EXPECT_EQ(l, iter->first); + ++iter; + }).unsafe_get0(); + } + + void submit_transaction(test_transaction_t t) { + tm->submit_transaction(std::move(t.t)).unsafe_get(); + test_mappings = t.mappings; + } +}; + +TEST_F(transaction_manager_test_t, basic) +{ + constexpr laddr_t SIZE = 4096; + run_async([this] { + constexpr laddr_t ADDR = 0xFF * SIZE; + { + auto t = create_transaction(); + auto extent = alloc_extent( + t, + ADDR, + SIZE, + 'a'); + ASSERT_EQ(ADDR, extent->get_laddr()); + check_mappings(t); + check(); + submit_transaction(std::move(t)); + check(); + } + }); +} + +TEST_F(transaction_manager_test_t, mutate) +{ + constexpr laddr_t SIZE = 4096; + run_async([this] { + constexpr laddr_t ADDR = 0xFF * SIZE; + { + auto t = create_transaction(); + auto extent = alloc_extent( + t, + ADDR, + SIZE, + 'a'); + ASSERT_EQ(ADDR, extent->get_laddr()); + check_mappings(t); + check(); + submit_transaction(std::move(t)); + check(); + } + ASSERT_TRUE(check_usage()); + replay(); + { + auto t = create_transaction(); + auto ext = get_extent( + t, + ADDR, + SIZE); + auto mut = mutate_extent(t, ext); + check_mappings(t); + check(); + submit_transaction(std::move(t)); + check(); + } + ASSERT_TRUE(check_usage()); + replay(); + check(); + }); +} + +TEST_F(transaction_manager_test_t, create_remove_same_transaction) +{ + constexpr laddr_t SIZE = 4096; + run_async([this] { + constexpr laddr_t ADDR = 0xFF * SIZE; + { + auto t = create_transaction(); + auto extent = alloc_extent( + t, + ADDR, + SIZE, + 'a'); + ASSERT_EQ(ADDR, extent->get_laddr()); + check_mappings(t); + dec_ref(t, ADDR); + check_mappings(t); + + extent = alloc_extent( + t, + ADDR, + SIZE, + 'a'); + + submit_transaction(std::move(t)); + check(); + } + replay(); + check(); + }); +} + +TEST_F(transaction_manager_test_t, split_merge_read_same_transaction) +{ + constexpr laddr_t SIZE = 4096; + run_async([this] { + { + auto t = create_transaction(); + for (unsigned i = 0; i < 300; ++i) { + auto extent = alloc_extent( + t, + laddr_t(i * SIZE), + SIZE); + } + check_mappings(t); + submit_transaction(std::move(t)); + check(); + } + { + auto t = create_transaction(); + for (unsigned i = 0; i < 240; ++i) { + dec_ref( + t, + laddr_t(i * SIZE)); + } + check_mappings(t); + submit_transaction(std::move(t)); + check(); + } + }); +} + + +TEST_F(transaction_manager_test_t, inc_dec_ref) +{ + constexpr laddr_t SIZE = 4096; + run_async([this] { + constexpr laddr_t ADDR = 0xFF * SIZE; + { + auto t = create_transaction(); + auto extent = alloc_extent( + t, + ADDR, + SIZE, + 'a'); + ASSERT_EQ(ADDR, extent->get_laddr()); + check_mappings(t); + check(); + submit_transaction(std::move(t)); + check(); + } + replay(); + { + auto t = create_transaction(); + inc_ref(t, ADDR); + check_mappings(t); + check(); + submit_transaction(std::move(t)); + check(); + } + { + auto t = create_transaction(); + dec_ref(t, ADDR); + check_mappings(t); + check(); + submit_transaction(std::move(t)); + check(); + } + replay(); + { + auto t = create_transaction(); + dec_ref(t, ADDR); + check_mappings(t); + check(); + submit_transaction(std::move(t)); + check(); + } + }); +} + +TEST_F(transaction_manager_test_t, cause_lba_split) +{ + constexpr laddr_t SIZE = 4096; + run_async([this] { + for (unsigned i = 0; i < 200; ++i) { + auto t = create_transaction(); + auto extent = alloc_extent( + t, + i * SIZE, + SIZE, + (char)(i & 0xFF)); + ASSERT_EQ(i * SIZE, extent->get_laddr()); + submit_transaction(std::move(t)); + } + check(); + }); +} + +TEST_F(transaction_manager_test_t, random_writes) +{ + constexpr size_t TOTAL = 4<<20; + constexpr size_t BSIZE = 4<<10; + constexpr size_t PADDING_SIZE = 256<<10; + constexpr size_t BLOCKS = TOTAL / BSIZE; + run_async([this] { + for (unsigned i = 0; i < BLOCKS; ++i) { + auto t = create_transaction(); + auto extent = alloc_extent( + t, + i * BSIZE, + BSIZE); + ASSERT_EQ(i * BSIZE, extent->get_laddr()); + submit_transaction(std::move(t)); + } + + for (unsigned i = 0; i < 4; ++i) { + for (unsigned j = 0; j < 65; ++j) { + auto t = create_transaction(); + for (unsigned k = 0; k < 2; ++k) { + auto ext = get_extent( + t, + get_random_laddr(BSIZE, TOTAL), + BSIZE); + auto mut = mutate_extent(t, ext); + // pad out transaction + auto padding = alloc_extent( + t, + TOTAL + (k * PADDING_SIZE), + PADDING_SIZE); + dec_ref(t, padding->get_laddr()); + } + submit_transaction(std::move(t)); + } + replay(); + logger().debug("random_writes: checking"); + check(); + logger().debug("random_writes: done replaying/checking"); + } + }); +} diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h new file mode 100644 index 000000000..bf2440923 --- /dev/null +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -0,0 +1,82 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include <random> + +#include "crimson/os/seastore/segment_cleaner.h" +#include "crimson/os/seastore/cache.h" +#include "crimson/os/seastore/transaction_manager.h" +#include "crimson/os/seastore/segment_manager/ephemeral.h" +#include "crimson/os/seastore/segment_manager.h" + +using namespace crimson; +using namespace crimson::os; +using namespace crimson::os::seastore; + +class TMTestState { +protected: + std::unique_ptr<segment_manager::EphemeralSegmentManager> segment_manager; + std::unique_ptr<SegmentCleaner> segment_cleaner; + std::unique_ptr<Journal> journal; + std::unique_ptr<Cache> cache; + LBAManagerRef lba_manager; + std::unique_ptr<TransactionManager> tm; + + TMTestState() + : segment_manager(segment_manager::create_test_ephemeral()) { + init(); + } + + void init() { + segment_cleaner = std::make_unique<SegmentCleaner>( + SegmentCleaner::config_t::default_from_segment_manager( + *segment_manager), + true); + journal = std::make_unique<Journal>(*segment_manager); + cache = std::make_unique<Cache>(*segment_manager); + lba_manager = lba_manager::create_lba_manager(*segment_manager, *cache); + tm = std::make_unique<TransactionManager>( + *segment_manager, *segment_cleaner, *journal, *cache, *lba_manager); + + journal->set_segment_provider(&*segment_cleaner); + segment_cleaner->set_extent_callback(&*tm); + } + + void destroy() { + tm.reset(); + lba_manager.reset(); + cache.reset(); + journal.reset(); + segment_cleaner.reset(); + } + + void restart() { + tm->close().unsafe_get(); + destroy(); + static_cast<segment_manager::EphemeralSegmentManager*>(&*segment_manager)->remount(); + init(); + tm->mount().unsafe_get(); + } + + seastar::future<> tm_setup() { + return segment_manager->init( + ).safe_then([this] { + return tm->mkfs(); + }).safe_then([this] { + return tm->close(); + }).safe_then([this] { + destroy(); + static_cast<segment_manager::EphemeralSegmentManager*>( + &*segment_manager)->remount(); + init(); + return tm->mount(); + }).handle_error(crimson::ct_error::assert_all{}); + } + + seastar::future<> tm_teardown() { + return tm->close( + ).handle_error(crimson::ct_error::assert_all{}); + } +}; diff --git a/src/test/crimson/test_alien_echo.cc b/src/test/crimson/test_alien_echo.cc new file mode 100644 index 000000000..4434b522b --- /dev/null +++ b/src/test/crimson/test_alien_echo.cc @@ -0,0 +1,306 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- + +#include "auth/Auth.h" +#include "messages/MPing.h" +#include "common/ceph_argparse.h" +#include "crimson/auth/DummyAuth.h" +#include "crimson/common/throttle.h" +#include "crimson/net/Connection.h" +#include "crimson/net/Dispatcher.h" +#include "crimson/net/Messenger.h" + +#include <seastar/core/alien.hh> +#include <seastar/core/app-template.hh> +#include <seastar/core/future-util.hh> +#include <seastar/core/internal/pollable_fd.hh> +#include <seastar/core/posix.hh> +#include <seastar/core/reactor.hh> + +using crimson::common::local_conf; + +enum class echo_role { + as_server, + as_client, +}; + +namespace seastar_pingpong { +struct DummyAuthAuthorizer : public AuthAuthorizer { + DummyAuthAuthorizer() + : AuthAuthorizer(CEPH_AUTH_CEPHX) + {} + bool verify_reply(bufferlist::const_iterator&, + std::string *connection_secret) override { + return true; + } + bool add_challenge(CephContext*, const bufferlist&) override { + return true; + } +}; + +struct Server { + crimson::common::Throttle byte_throttler; + crimson::net::MessengerRef msgr; + crimson::auth::DummyAuthClientServer dummy_auth; + struct ServerDispatcher final : crimson::net::Dispatcher { + unsigned count = 0; + seastar::condition_variable on_reply; + std::optional<seastar::future<>> ms_dispatch(crimson::net::ConnectionRef c, + MessageRef m) final + { + std::cout << "server got ping " << *m << std::endl; + // reply with a pong + return c->send(make_message<MPing>()).then([this] { + ++count; + on_reply.signal(); + return seastar::now(); + }); + } + } dispatcher; + Server(crimson::net::MessengerRef msgr) + : byte_throttler(local_conf()->osd_client_message_size_cap), + msgr{msgr} + { + msgr->set_crc_header(); + msgr->set_crc_data(); + } +}; + +struct Client { + crimson::common::Throttle byte_throttler; + crimson::net::MessengerRef msgr; + crimson::auth::DummyAuthClientServer dummy_auth; + struct ClientDispatcher final : crimson::net::Dispatcher { + unsigned count = 0; + seastar::condition_variable on_reply; + std::optional<seastar::future<>> ms_dispatch(crimson::net::ConnectionRef c, + MessageRef m) final + { + std::cout << "client got pong " << *m << std::endl; + ++count; + on_reply.signal(); + return seastar::now(); + } + } dispatcher; + Client(crimson::net::MessengerRef msgr) + : byte_throttler(local_conf()->osd_client_message_size_cap), + msgr{msgr} + { + msgr->set_crc_header(); + msgr->set_crc_data(); + } +}; +} // namespace seastar_pingpong + +class SeastarContext { + int begin_fd; + seastar::file_desc on_end; + +public: + SeastarContext() + : begin_fd{eventfd(0, 0)}, + on_end{seastar::file_desc::eventfd(0, 0)} + {} + + template<class Func> + std::thread with_seastar(Func&& func) { + return std::thread{[this, on_end = on_end.get(), + func = std::forward<Func>(func)] { + // alien: are you ready? + wait_for_seastar(); + // alien: could you help me apply(func)? + func(); + // alien: i've sent my request. have you replied it? + // wait_for_seastar(); + // alien: you are free to go! + ::eventfd_write(on_end, 1); + }}; + } + + void run(seastar::app_template& app, int argc, char** argv) { + app.run(argc, argv, [this] { + std::vector<const char*> args; + std::string cluster; + std::string conf_file_list; + auto init_params = ceph_argparse_early_args(args, + CEPH_ENTITY_TYPE_CLIENT, + &cluster, + &conf_file_list); + return crimson::common::sharded_conf().start(init_params.name, cluster) + .then([conf_file_list] { + return local_conf().parse_config_files(conf_file_list); + }).then([this] { + return set_seastar_ready(); + }).then([on_end = std::move(on_end)] () mutable { + // seastar: let me know once i am free to leave. + return seastar::do_with(seastar::pollable_fd(std::move(on_end)), [] + (seastar::pollable_fd& on_end_fds) { + return on_end_fds.readable().then([&on_end_fds] { + eventfd_t result = 0; + on_end_fds.get_file_desc().read(&result, sizeof(result)); + return seastar::make_ready_future<>(); + }); + }); + }).then([]() { + return crimson::common::sharded_conf().stop(); + }).handle_exception([](auto ep) { + std::cerr << "Error: " << ep << std::endl; + }).finally([] { + seastar::engine().exit(0); + }); + }); + } + + seastar::future<> set_seastar_ready() { + // seastar: i am ready to serve! + ::eventfd_write(begin_fd, 1); + return seastar::now(); + } + +private: + void wait_for_seastar() { + eventfd_t result = 0; + if (int r = ::eventfd_read(begin_fd, &result); r < 0) { + std::cerr << "unable to eventfd_read():" << errno << std::endl; + } + } +}; + +static seastar::future<> +seastar_echo(const entity_addr_t addr, echo_role role, unsigned count) +{ + std::cout << "seastar/"; + if (role == echo_role::as_server) { + return seastar::do_with( + seastar_pingpong::Server{crimson::net::Messenger::create( + entity_name_t::OSD(0), "server", addr.get_nonce())}, + [addr, count](auto& server) mutable { + std::cout << "server listening at " << addr << std::endl; + // bind the server + server.msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); + server.msgr->set_policy_throttler(entity_name_t::TYPE_OSD, + &server.byte_throttler); + server.msgr->set_require_authorizer(false); + server.msgr->set_auth_client(&server.dummy_auth); + server.msgr->set_auth_server(&server.dummy_auth); + return server.msgr->bind(entity_addrvec_t{addr} + ).safe_then([&server] { + return server.msgr->start({&server.dispatcher}); + }, crimson::net::Messenger::bind_ertr::all_same_way([](auto& e) { + ceph_abort_msg("bind failed"); + })).then([&dispatcher=server.dispatcher, count] { + return dispatcher.on_reply.wait([&dispatcher, count] { + return dispatcher.count >= count; + }); + }).finally([&server] { + std::cout << "server shutting down" << std::endl; + return server.msgr->shutdown(); + }); + }); + } else { + return seastar::do_with( + seastar_pingpong::Client{crimson::net::Messenger::create( + entity_name_t::OSD(1), "client", addr.get_nonce())}, + [addr, count](auto& client) { + std::cout << "client sending to " << addr << std::endl; + client.msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); + client.msgr->set_policy_throttler(entity_name_t::TYPE_OSD, + &client.byte_throttler); + client.msgr->set_require_authorizer(false); + client.msgr->set_auth_client(&client.dummy_auth); + client.msgr->set_auth_server(&client.dummy_auth); + return client.msgr->start({&client.dispatcher}).then( + [addr, &client, &disp=client.dispatcher, count] { + auto conn = client.msgr->connect(addr, entity_name_t::TYPE_OSD); + return seastar::do_until( + [&disp,count] { return disp.count >= count; }, + [&disp,conn] { + return conn->send(make_message<MPing>()).then([&] { + return disp.on_reply.wait(); + }); + } + ); + }).finally([&client] { + std::cout << "client shutting down" << std::endl; + return client.msgr->shutdown(); + }); + }); + } +} + +int main(int argc, char** argv) +{ + namespace po = boost::program_options; + po::options_description desc{"Allowed options"}; + desc.add_options() + ("help,h", "show help message") + ("role", po::value<std::string>()->default_value("pong"), + "role to play (ping | pong)") + ("port", po::value<uint16_t>()->default_value(9010), + "port #") + ("nonce", po::value<uint32_t>()->default_value(42), + "a unique number to identify the pong server") + ("count", po::value<unsigned>()->default_value(10), + "stop after sending/echoing <count> MPing messages") + ("v2", po::value<bool>()->default_value(false), + "using msgr v2 protocol"); + po::variables_map vm; + std::vector<std::string> unrecognized_options; + try { + auto parsed = po::command_line_parser(argc, argv) + .options(desc) + .allow_unregistered() + .run(); + po::store(parsed, vm); + if (vm.count("help")) { + std::cout << desc << std::endl; + return 0; + } + po::notify(vm); + unrecognized_options = po::collect_unrecognized(parsed.options, po::include_positional); + } catch(const po::error& e) { + std::cerr << "error: " << e.what() << std::endl; + return 1; + } + + entity_addr_t addr; + if (vm["v2"].as<bool>()) { + addr.set_type(entity_addr_t::TYPE_MSGR2); + } else { + addr.set_type(entity_addr_t::TYPE_LEGACY); + } + addr.set_family(AF_INET); + addr.set_port(vm["port"].as<std::uint16_t>()); + addr.set_nonce(vm["nonce"].as<std::uint32_t>()); + + echo_role role = echo_role::as_server; + if (vm["role"].as<std::string>() == "ping") { + role = echo_role::as_client; + } + + auto count = vm["count"].as<unsigned>(); + seastar::app_template app; + SeastarContext sc; + auto job = sc.with_seastar([&] { + auto fut = seastar::alien::submit_to(0, [addr, role, count] { + return seastar_echo(addr, role, count); + }); + fut.wait(); + }); + std::vector<char*> av{argv[0]}; + std::transform(begin(unrecognized_options), + end(unrecognized_options), + std::back_inserter(av), + [](auto& s) { + return const_cast<char*>(s.c_str()); + }); + sc.run(app, av.size(), av.data()); + job.join(); +} + +/* + * Local Variables: + * compile-command: "make -j4 \ + * -C ../../../build \ + * unittest_seastar_echo" + * End: + */ diff --git a/src/test/crimson/test_alienstore_thread_pool.cc b/src/test/crimson/test_alienstore_thread_pool.cc new file mode 100644 index 000000000..82b98abbb --- /dev/null +++ b/src/test/crimson/test_alienstore_thread_pool.cc @@ -0,0 +1,78 @@ +#include <chrono> +#include <iostream> +#include <numeric> +#include <seastar/core/app-template.hh> +#include "common/ceph_argparse.h" +#include "crimson/common/config_proxy.h" +#include "crimson/os/alienstore/thread_pool.h" +#include "include/msgr.h" + +using namespace std::chrono_literals; +using ThreadPool = crimson::os::ThreadPool; +using crimson::common::local_conf; + +seastar::future<> test_accumulate(ThreadPool& tp) { + static constexpr auto N = 5; + static constexpr auto M = 1; + auto slow_plus = [&tp](int i) { + return tp.submit([=] { + std::this_thread::sleep_for(10ns); + return i + M; + }); + }; + return seastar::map_reduce( + boost::irange(0, N), slow_plus, 0, std::plus{}).then([] (int sum) { + auto r = boost::irange(0 + M, N + M); + if (sum != std::accumulate(r.begin(), r.end(), 0)) { + throw std::runtime_error("test_accumulate failed"); + } + }); +} + +seastar::future<> test_void_return(ThreadPool& tp) { + return tp.submit([=] { + std::this_thread::sleep_for(10ns); + }); +} + +int main(int argc, char** argv) +{ + seastar::app_template app; + return app.run(argc, argv, [] { + std::vector<const char*> args; + std::string cluster; + std::string conf_file_list; + auto init_params = ceph_argparse_early_args(args, + CEPH_ENTITY_TYPE_CLIENT, + &cluster, + &conf_file_list); + return crimson::common::sharded_conf().start(init_params.name, cluster) + .then([conf_file_list] { + return local_conf().parse_config_files(conf_file_list); + }).then([] { + return seastar::do_with(std::make_unique<crimson::os::ThreadPool>(2, 128, 0), + [](auto& tp) { + return tp->start().then([&tp] { + return test_accumulate(*tp); + }).then([&tp] { + return test_void_return(*tp); + }).finally([&tp] { + return tp->stop(); + }); + }); + }).finally([] { + return crimson::common::sharded_conf().stop(); + }).handle_exception([](auto e) { + std::cerr << "Error: " << e << std::endl; + seastar::engine().exit(1); + }); + }); +} + +/* + * Local Variables: + * compile-command: "make -j4 \ + * -C ../../../build \ + * unittest_seastar_thread_pool" + * End: + */ diff --git a/src/test/crimson/test_async_echo.cc b/src/test/crimson/test_async_echo.cc new file mode 100644 index 000000000..4f2d43d9d --- /dev/null +++ b/src/test/crimson/test_async_echo.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- + +#include <boost/program_options/variables_map.hpp> +#include <boost/program_options/parsers.hpp> + +#include "auth/Auth.h" +#include "global/global_init.h" +#include "messages/MPing.h" +#include "msg/Dispatcher.h" +#include "msg/Messenger.h" + +#include "auth/DummyAuth.h" + +enum class echo_role { + as_server, + as_client, +}; + +namespace native_pingpong { + +constexpr int CEPH_OSD_PROTOCOL = 10; + +struct Server { + Server(CephContext* cct, const entity_inst_t& entity) + : dummy_auth(cct), dispatcher(cct) + { + msgr.reset(Messenger::create(cct, "async", entity.name, "pong", entity.addr.get_nonce())); + dummy_auth.auth_registry.refresh_config(); + msgr->set_cluster_protocol(CEPH_OSD_PROTOCOL); + msgr->set_default_policy(Messenger::Policy::stateless_server(0)); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + msgr->set_require_authorizer(false); + } + DummyAuthClientServer dummy_auth; + unique_ptr<Messenger> msgr; + struct ServerDispatcher : Dispatcher { + std::mutex mutex; + std::condition_variable on_reply; + bool replied = false; + ServerDispatcher(CephContext* cct) + : Dispatcher(cct) + {} + bool ms_can_fast_dispatch_any() const override { + return true; + } + bool ms_can_fast_dispatch(const Message* m) const override { + return m->get_type() == CEPH_MSG_PING; + } + void ms_fast_dispatch(Message* m) override { + m->get_connection()->send_message(new MPing); + m->put(); + { + std::lock_guard lock{mutex}; + replied = true; + } + on_reply.notify_one(); + } + bool ms_dispatch(Message*) override { + ceph_abort(); + } + bool ms_handle_reset(Connection*) override { + return true; + } + void ms_handle_remote_reset(Connection*) override { + } + bool ms_handle_refused(Connection*) override { + return true; + } + void echo() { + replied = false; + std::unique_lock lock{mutex}; + return on_reply.wait(lock, [this] { return replied; }); + } + } dispatcher; + void echo() { + dispatcher.echo(); + } +}; + +struct Client { + unique_ptr<Messenger> msgr; + Client(CephContext *cct) + : dummy_auth(cct), dispatcher(cct) + { + msgr.reset(Messenger::create(cct, "async", entity_name_t::CLIENT(-1), "ping", getpid())); + dummy_auth.auth_registry.refresh_config(); + msgr->set_cluster_protocol(CEPH_OSD_PROTOCOL); + msgr->set_default_policy(Messenger::Policy::lossy_client(0)); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + msgr->set_require_authorizer(false); + } + DummyAuthClientServer dummy_auth; + struct ClientDispatcher : Dispatcher { + std::mutex mutex; + std::condition_variable on_reply; + bool replied = false; + + ClientDispatcher(CephContext* cct) + : Dispatcher(cct) + {} + bool ms_can_fast_dispatch_any() const override { + return true; + } + bool ms_can_fast_dispatch(const Message* m) const override { + return m->get_type() == CEPH_MSG_PING; + } + void ms_fast_dispatch(Message* m) override { + m->put(); + { + std::lock_guard lock{mutex}; + replied = true; + } + on_reply.notify_one(); + } + bool ms_dispatch(Message*) override { + ceph_abort(); + } + bool ms_handle_reset(Connection *) override { + return true; + } + void ms_handle_remote_reset(Connection*) override { + } + bool ms_handle_refused(Connection*) override { + return true; + } + bool ping(Messenger* msgr, const entity_inst_t& peer) { + auto conn = msgr->connect_to(peer.name.type(), + entity_addrvec_t{peer.addr}); + replied = false; + conn->send_message(new MPing); + std::unique_lock lock{mutex}; + return on_reply.wait_for(lock, 500ms, [&] { + return replied; + }); + } + } dispatcher; + void ping(const entity_inst_t& peer) { + dispatcher.ping(msgr.get(), peer); + } +}; +} // namespace native_pingpong + +static void ceph_echo(CephContext* cct, + entity_addr_t addr, echo_role role, unsigned count) +{ + std::cout << "ceph/"; + entity_inst_t entity{entity_name_t::OSD(0), addr}; + if (role == echo_role::as_server) { + std::cout << "server listening at " << addr << std::endl; + native_pingpong::Server server{cct, entity}; + server.msgr->bind(addr); + server.msgr->add_dispatcher_head(&server.dispatcher); + server.msgr->start(); + for (unsigned i = 0; i < count; i++) { + server.echo(); + } + server.msgr->shutdown(); + server.msgr->wait(); + } else { + std::cout << "client sending to " << addr << std::endl; + native_pingpong::Client client{cct}; + client.msgr->add_dispatcher_head(&client.dispatcher); + client.msgr->start(); + auto conn = client.msgr->connect_to(entity.name.type(), + entity_addrvec_t{entity.addr}); + for (unsigned i = 0; i < count; i++) { + std::cout << "seq=" << i << std::endl; + client.ping(entity); + } + client.msgr->shutdown(); + client.msgr->wait(); + } +} + +int main(int argc, char** argv) +{ + namespace po = boost::program_options; + po::options_description desc{"Allowed options"}; + desc.add_options() + ("help,h", "show help message") + ("role", po::value<std::string>()->default_value("pong"), + "role to play (ping | pong)") + ("port", po::value<uint16_t>()->default_value(9010), + "port #") + ("nonce", po::value<uint32_t>()->default_value(42), + "a unique number to identify the pong server") + ("count", po::value<unsigned>()->default_value(10), + "stop after sending/echoing <count> MPing messages") + ("v2", po::value<bool>()->default_value(false), + "using msgr v2 protocol"); + po::variables_map vm; + std::vector<std::string> unrecognized_options; + try { + auto parsed = po::command_line_parser(argc, argv) + .options(desc) + .allow_unregistered() + .run(); + po::store(parsed, vm); + if (vm.count("help")) { + std::cout << desc << std::endl; + return 0; + } + po::notify(vm); + unrecognized_options = po::collect_unrecognized(parsed.options, po::include_positional); + } catch(const po::error& e) { + std::cerr << "error: " << e.what() << std::endl; + return 1; + } + + entity_addr_t addr; + if (vm["v2"].as<bool>()) { + addr.set_type(entity_addr_t::TYPE_MSGR2); + } else { + addr.set_type(entity_addr_t::TYPE_LEGACY); + } + addr.set_family(AF_INET); + addr.set_port(vm["port"].as<std::uint16_t>()); + addr.set_nonce(vm["nonce"].as<std::uint32_t>()); + + echo_role role = echo_role::as_server; + if (vm["role"].as<std::string>() == "ping") { + role = echo_role::as_client; + } + + auto count = vm["count"].as<unsigned>(); + std::vector<const char*> args(argv, argv + argc); + auto cct = global_init(nullptr, args, + CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_MON_CONFIG); + common_init_finish(cct.get()); + ceph_echo(cct.get(), addr, role, count); +} diff --git a/src/test/crimson/test_backfill.cc b/src/test/crimson/test_backfill.cc new file mode 100644 index 000000000..8f3bc0d9b --- /dev/null +++ b/src/test/crimson/test_backfill.cc @@ -0,0 +1,500 @@ +#include <algorithm> +#include <cstdlib> +#include <deque> +#include <functional> +#include <initializer_list> +#include <iostream> +#include <iterator> +#include <limits> +#include <map> +#include <set> +#include <string> + +#include <boost/statechart/event_base.hpp> +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +#include "common/hobject.h" +#include "crimson/osd/backfill_state.h" +#include "osd/recovery_types.h" + + +// The sole purpose is to convert from the string representation. +// An alternative approach could use boost::range in FakeStore's +// constructor. +struct improved_hobject_t : hobject_t { + improved_hobject_t(const char parsable_name[]) { + this->parse(parsable_name); + } + improved_hobject_t(const hobject_t& obj) + : hobject_t(obj) { + } + bool operator==(const improved_hobject_t& rhs) const { + return static_cast<const hobject_t&>(*this) == \ + static_cast<const hobject_t&>(rhs); + } +}; + + +struct FakeStore { + using objs_t = std::map<improved_hobject_t, eversion_t>; + + objs_t objs; + + void push(const hobject_t& obj, eversion_t version) { + objs[obj] = version; + } + + void drop(const hobject_t& obj, const eversion_t version) { + auto it = objs.find(obj); + ceph_assert(it != std::end(objs)); + ceph_assert(it->second == version); + objs.erase(it); + } + + template <class Func> + hobject_t list(const hobject_t& start, Func&& per_entry) const { + auto it = objs.lower_bound(start); + for (auto max = std::numeric_limits<std::uint64_t>::max(); + it != std::end(objs) && max > 0; + ++it, --max) { + per_entry(*it); + } + return it != std::end(objs) ? static_cast<const hobject_t&>(it->first) + : hobject_t::get_max(); + } + + bool operator==(const FakeStore& rhs) const { + return std::size(objs) == std::size(rhs.objs) && \ + std::equal(std::begin(objs), std::end(objs), std::begin(rhs.objs)); + } + bool operator!=(const FakeStore& rhs) const { + return !(*this == rhs); + } +}; + + +struct FakeReplica { + FakeStore store; + hobject_t last_backfill; + + FakeReplica(FakeStore&& store) + : store(std::move(store)) { + } +}; + +struct FakePrimary { + FakeStore store; + eversion_t last_update; + eversion_t projected_last_update; + eversion_t log_tail; + + FakePrimary(FakeStore&& store) + : store(std::move(store)) { + } +}; + +class BackfillFixture : public crimson::osd::BackfillState::BackfillListener { + friend class BackfillFixtureBuilder; + + FakePrimary backfill_source; + std::map<pg_shard_t, FakeReplica> backfill_targets; + std::map<pg_shard_t, + std::vector<std::pair<hobject_t, eversion_t>>> enqueued_drops; + std::deque< + boost::intrusive_ptr< + const boost::statechart::event_base>> events_to_dispatch; + crimson::osd::BackfillState backfill_state; + + BackfillFixture(FakePrimary&& backfill_source, + std::map<pg_shard_t, FakeReplica>&& backfill_targets); + + template <class EventT> + void schedule_event(const EventT& event) { + events_to_dispatch.emplace_back(event.intrusive_from_this()); + } + + // BackfillListener { + void request_replica_scan( + const pg_shard_t& target, + const hobject_t& begin, + const hobject_t& end) override; + + void request_primary_scan( + const hobject_t& begin) override; + + void enqueue_push( + const hobject_t& obj, + const eversion_t& v) override; + + void enqueue_drop( + const pg_shard_t& target, + const hobject_t& obj, + const eversion_t& v) override; + + void maybe_flush() override; + + void update_peers_last_backfill( + const hobject_t& new_last_backfill) override; + + bool budget_available() const override; + +public: + MOCK_METHOD(void, backfilled, (), (override)); + // } + + void next_round(std::size_t how_many=1) { + ceph_assert(events_to_dispatch.size() >= how_many); + while (how_many-- > 0) { + backfill_state.process_event(std::move(events_to_dispatch.front())); + events_to_dispatch.pop_front(); + } + } + + void next_till_done() { + while (!events_to_dispatch.empty()) { + next_round(); + } + } + + bool all_stores_look_like(const FakeStore& reference) const { + const bool all_replica_match = std::all_of( + std::begin(backfill_targets), std::end(backfill_targets), + [&reference] (const auto kv) { + return kv.second.store == reference; + }); + return backfill_source.store == reference && all_replica_match; + } + + struct PeeringFacade; + struct PGFacade; +}; + +struct BackfillFixture::PeeringFacade + : public crimson::osd::BackfillState::PeeringFacade { + FakePrimary& backfill_source; + std::map<pg_shard_t, FakeReplica>& backfill_targets; + // sorry, this is duplicative but that's the interface + std::set<pg_shard_t> backfill_targets_as_set; + + PeeringFacade(FakePrimary& backfill_source, + std::map<pg_shard_t, FakeReplica>& backfill_targets) + : backfill_source(backfill_source), + backfill_targets(backfill_targets) { + std::transform( + std::begin(backfill_targets), std::end(backfill_targets), + std::inserter(backfill_targets_as_set, std::end(backfill_targets_as_set)), + [](auto pair) { + return pair.first; + }); + } + + hobject_t earliest_backfill() const override { + hobject_t e = hobject_t::get_max(); + for (const auto& kv : backfill_targets) { + e = std::min(kv.second.last_backfill, e); + } + return e; + } + const std::set<pg_shard_t>& get_backfill_targets() const override { + return backfill_targets_as_set; + } + const hobject_t& get_peer_last_backfill(pg_shard_t peer) const override { + return backfill_targets.at(peer).last_backfill; + } + const eversion_t& get_last_update() const override { + return backfill_source.last_update; + } + const eversion_t& get_log_tail() const override { + return backfill_source.log_tail; + } + + void scan_log_after(eversion_t, scan_log_func_t) const override { + /* NOP */ + } + + bool is_backfill_target(pg_shard_t peer) const override { + return backfill_targets.count(peer) == 1; + } + void update_complete_backfill_object_stats(const hobject_t &hoid, + const pg_stat_t &stats) override { + } + bool is_backfilling() const override { + return true; + } +}; + +struct BackfillFixture::PGFacade : public crimson::osd::BackfillState::PGFacade { + FakePrimary& backfill_source; + + PGFacade(FakePrimary& backfill_source) + : backfill_source(backfill_source) { + } + + const eversion_t& get_projected_last_update() const override { + return backfill_source.projected_last_update; + } +}; + +BackfillFixture::BackfillFixture( + FakePrimary&& backfill_source, + std::map<pg_shard_t, FakeReplica>&& backfill_targets) + : backfill_source(std::move(backfill_source)), + backfill_targets(std::move(backfill_targets)), + backfill_state(*this, + std::make_unique<PeeringFacade>(this->backfill_source, + this->backfill_targets), + std::make_unique<PGFacade>(this->backfill_source)) +{ + backfill_state.process_event(crimson::osd::BackfillState::Triggered{}.intrusive_from_this()); +} + +void BackfillFixture::request_replica_scan( + const pg_shard_t& target, + const hobject_t& begin, + const hobject_t& end) +{ + BackfillInterval bi; + bi.end = backfill_targets.at(target).store.list(begin, [&bi](auto kv) { + bi.objects.insert(std::move(kv)); + }); + bi.begin = begin; + bi.version = backfill_source.last_update; + + schedule_event(crimson::osd::BackfillState::ReplicaScanned{ target, std::move(bi) }); +} + +void BackfillFixture::request_primary_scan( + const hobject_t& begin) +{ + BackfillInterval bi; + bi.end = backfill_source.store.list(begin, [&bi](auto kv) { + bi.objects.insert(std::move(kv)); + }); + bi.begin = begin; + bi.version = backfill_source.last_update; + + schedule_event(crimson::osd::BackfillState::PrimaryScanned{ std::move(bi) }); +} + +void BackfillFixture::enqueue_push( + const hobject_t& obj, + const eversion_t& v) +{ + for (auto& [ _, bt ] : backfill_targets) { + bt.store.push(obj, v); + } + schedule_event(crimson::osd::BackfillState::ObjectPushed{ obj }); +} + +void BackfillFixture::enqueue_drop( + const pg_shard_t& target, + const hobject_t& obj, + const eversion_t& v) +{ + enqueued_drops[target].emplace_back(obj, v); +} + +void BackfillFixture::maybe_flush() +{ + for (const auto& [target, versioned_objs] : enqueued_drops) { + for (const auto& [obj, v] : versioned_objs) { + backfill_targets.at(target).store.drop(obj, v); + } + } + enqueued_drops.clear(); +} + +void BackfillFixture::update_peers_last_backfill( + const hobject_t& new_last_backfill) +{ +} + +bool BackfillFixture::budget_available() const +{ + return true; +} + +struct BackfillFixtureBuilder { + FakeStore backfill_source; + std::map<pg_shard_t, FakeReplica> backfill_targets; + + static BackfillFixtureBuilder add_source(FakeStore::objs_t objs) { + BackfillFixtureBuilder bfb; + bfb.backfill_source = FakeStore{ std::move(objs) }; + return bfb; + } + + BackfillFixtureBuilder&& add_target(FakeStore::objs_t objs) && { + const auto new_osd_num = std::size(backfill_targets); + const auto [ _, inserted ] = backfill_targets.emplace( + new_osd_num, FakeReplica{ FakeStore{std::move(objs)} }); + ceph_assert(inserted); + return std::move(*this); + } + + BackfillFixture get_result() && { + return BackfillFixture{ std::move(backfill_source), + std::move(backfill_targets) }; + } +}; + +// The straightest case: single primary, single replica. All have the same +// content in their object stores, so the entire backfill boils into just +// `request_primary_scan()` and `request_replica_scan()`. +TEST(backfill, same_primary_same_replica) +{ + const auto reference_store = FakeStore{ { + { "1:00058bcc:::rbd_data.1018ac3e755.00000000000000d5:head", {10, 234} }, + { "1:00ed7f8e:::rbd_data.1018ac3e755.00000000000000af:head", {10, 196} }, + { "1:01483aea:::rbd_data.1018ac3e755.0000000000000095:head", {10, 169} }, + }}; + auto cluster_fixture = BackfillFixtureBuilder::add_source( + reference_store.objs + ).add_target( + reference_store.objs + ).get_result(); + + cluster_fixture.next_round(); + EXPECT_CALL(cluster_fixture, backfilled); + cluster_fixture.next_round(); + EXPECT_TRUE(cluster_fixture.all_stores_look_like(reference_store)); +} + +TEST(backfill, one_empty_replica) +{ + const auto reference_store = FakeStore{ { + { "1:00058bcc:::rbd_data.1018ac3e755.00000000000000d5:head", {10, 234} }, + { "1:00ed7f8e:::rbd_data.1018ac3e755.00000000000000af:head", {10, 196} }, + { "1:01483aea:::rbd_data.1018ac3e755.0000000000000095:head", {10, 169} }, + }}; + auto cluster_fixture = BackfillFixtureBuilder::add_source( + reference_store.objs + ).add_target( + { /* nothing */ } + ).get_result(); + + cluster_fixture.next_round(); + cluster_fixture.next_round(); + cluster_fixture.next_round(2); + EXPECT_CALL(cluster_fixture, backfilled); + cluster_fixture.next_round(); + EXPECT_TRUE(cluster_fixture.all_stores_look_like(reference_store)); +} + +TEST(backfill, two_empty_replicas) +{ + const auto reference_store = FakeStore{ { + { "1:00058bcc:::rbd_data.1018ac3e755.00000000000000d5:head", {10, 234} }, + { "1:00ed7f8e:::rbd_data.1018ac3e755.00000000000000af:head", {10, 196} }, + { "1:01483aea:::rbd_data.1018ac3e755.0000000000000095:head", {10, 169} }, + }}; + auto cluster_fixture = BackfillFixtureBuilder::add_source( + reference_store.objs + ).add_target( + { /* nothing 1 */ } + ).add_target( + { /* nothing 2 */ } + ).get_result(); + + EXPECT_CALL(cluster_fixture, backfilled); + cluster_fixture.next_till_done(); + + EXPECT_TRUE(cluster_fixture.all_stores_look_like(reference_store)); +} + +namespace StoreRandomizer { + // FIXME: copied & pasted from test/test_snap_mapper.cc. We need to + // find a way to avoid code duplication in test. A static library? + std::string random_string(std::size_t size) { + std::string name; + for (size_t j = 0; j < size; ++j) { + name.push_back('a' + (std::rand() % 26)); + } + return name; + } + + hobject_t random_hobject() { + uint32_t mask{0}; + uint32_t bits{0}; + return hobject_t( + random_string(1+(std::rand() % 16)), + random_string(1+(std::rand() % 16)), + snapid_t(std::rand() % 1000), + (std::rand() & ((~0)<<bits)) | (mask & ~((~0)<<bits)), + 0, random_string(std::rand() % 16)); + } + + eversion_t random_eversion() { + return eversion_t{ std::rand() % 512U, std::rand() % 256UL }; + } + + FakeStore create() { + FakeStore store; + for (std::size_t i = std::rand() % 2048; i > 0; --i) { + store.push(random_hobject(), random_eversion()); + } + return store; + } + + template <class... Args> + void execute_random(Args&&... args) { + std::array<std::function<void()>, sizeof...(Args)> funcs = { + std::forward<Args>(args)... + }; + return std::move(funcs[std::rand() % std::size(funcs)])(); + } + + FakeStore mutate(const FakeStore& source_store) { + FakeStore mutated_store; + source_store.list(hobject_t{}, [&] (const auto& kv) { + const auto& [ oid, version ] = kv; + execute_random( + [] { /* just drop the entry */ }, + [&] { mutated_store.push(oid, version); }, + [&] { mutated_store.push(oid, random_eversion()); }, + [&] { mutated_store.push(random_hobject(), version); }, + [&] { + for (auto how_many = std::rand() % 8; how_many > 0; --how_many) { + mutated_store.push(random_hobject(), random_eversion()); + } + } + ); + }); + return mutated_store; + } +} + +// The name might suggest randomness is involved here. Well, that's true +// but till we know the seed the test still is repeatable. +TEST(backfill, one_pseudorandomized_replica) +{ + const auto reference_store = StoreRandomizer::create(); + auto cluster_fixture = BackfillFixtureBuilder::add_source( + reference_store.objs + ).add_target( + StoreRandomizer::mutate(reference_store).objs + ).get_result(); + + EXPECT_CALL(cluster_fixture, backfilled); + cluster_fixture.next_till_done(); + + EXPECT_TRUE(cluster_fixture.all_stores_look_like(reference_store)); +} + +TEST(backfill, two_pseudorandomized_replicas) +{ + const auto reference_store = StoreRandomizer::create(); + auto cluster_fixture = BackfillFixtureBuilder::add_source( + reference_store.objs + ).add_target( + StoreRandomizer::mutate(reference_store).objs + ).add_target( + StoreRandomizer::mutate(reference_store).objs + ).get_result(); + + EXPECT_CALL(cluster_fixture, backfilled); + cluster_fixture.next_till_done(); + + EXPECT_TRUE(cluster_fixture.all_stores_look_like(reference_store)); +} diff --git a/src/test/crimson/test_buffer.cc b/src/test/crimson/test_buffer.cc new file mode 100644 index 000000000..64a815bd2 --- /dev/null +++ b/src/test/crimson/test_buffer.cc @@ -0,0 +1,50 @@ +#include <iostream> +#include <seastar/core/app-template.hh> +#include <seastar/core/future-util.hh> +#include <seastar/core/reactor.hh> +#include "include/buffer.h" + +// allocate a foreign buffer on each cpu, collect them all into a bufferlist, +// and destruct it on this cpu +seastar::future<> test_foreign_bufferlist() +{ + auto make_foreign_buffer = [] (unsigned cpu) { + return seastar::smp::submit_to(cpu, [=] { + bufferlist bl; + seastar::temporary_buffer<char> buf("abcd", 4); + bl.append(buffer::create(std::move(buf))); + return bl; + }); + }; + auto reduce = [] (bufferlist&& lhs, bufferlist&& rhs) { + bufferlist bl; + bl.claim_append(lhs); + bl.claim_append(rhs); + return bl; + }; + return seastar::map_reduce(seastar::smp::all_cpus(), make_foreign_buffer, + bufferlist(), reduce).then( + [] (bufferlist&& bl) { + if (bl.length() != 4 * seastar::smp::count) { + auto e = std::make_exception_ptr(std::runtime_error("wrong buffer size")); + return seastar::make_exception_future<>(e); + } + bl.clear(); + return seastar::make_ready_future<>(); + }); +} + +int main(int argc, char** argv) +{ + seastar::app_template app; + return app.run(argc, argv, [] { + return seastar::now().then( + &test_foreign_bufferlist + ).then([] { + std::cout << "All tests succeeded" << std::endl; + }).handle_exception([] (auto eptr) { + std::cout << "Test failure" << std::endl; + return seastar::make_exception_future<>(eptr); + }); + }); +} diff --git a/src/test/crimson/test_cmds.h b/src/test/crimson/test_cmds.h new file mode 100644 index 000000000..2320d30a0 --- /dev/null +++ b/src/test/crimson/test_cmds.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +namespace ceph::net::test { + +enum class cmd_t : char { + none = '\0', + shutdown, + suite_start, + suite_stop, + suite_connect_me, + suite_send_me, + suite_keepalive_me, + suite_markdown, + suite_recv_op +}; + +enum class policy_t : char { + none = '\0', + stateful_server, + stateless_server, + lossless_peer, + lossless_peer_reuse, + lossy_client, + lossless_client +}; + +inline std::ostream& operator<<(std::ostream& out, const cmd_t& cmd) { + switch(cmd) { + case cmd_t::none: + return out << "none"; + case cmd_t::shutdown: + return out << "shutdown"; + case cmd_t::suite_start: + return out << "suite_start"; + case cmd_t::suite_stop: + return out << "suite_stop"; + case cmd_t::suite_connect_me: + return out << "suite_connect_me"; + case cmd_t::suite_send_me: + return out << "suite_send_me"; + case cmd_t::suite_keepalive_me: + return out << "suite_keepalive_me"; + case cmd_t::suite_markdown: + return out << "suite_markdown"; + case cmd_t::suite_recv_op: + return out << "suite_recv_op"; + default: + ceph_abort(); + } +} + +inline std::ostream& operator<<(std::ostream& out, const policy_t& policy) { + switch(policy) { + case policy_t::none: + return out << "none"; + case policy_t::stateful_server: + return out << "stateful_server"; + case policy_t::stateless_server: + return out << "stateless_server"; + case policy_t::lossless_peer: + return out << "lossless_peer"; + case policy_t::lossless_peer_reuse: + return out << "lossless_peer_reuse"; + case policy_t::lossy_client: + return out << "lossy_client"; + case policy_t::lossless_client: + return out << "lossless_client"; + default: + ceph_abort(); + } +} + +} // namespace ceph::net::test diff --git a/src/test/crimson/test_config.cc b/src/test/crimson/test_config.cc new file mode 100644 index 000000000..608aa2694 --- /dev/null +++ b/src/test/crimson/test_config.cc @@ -0,0 +1,108 @@ +#include <chrono> +#include <string> +#include <numeric> +#include <seastar/core/app-template.hh> +#include <seastar/core/sharded.hh> +#include "common/ceph_argparse.h" +#include "common/config_obs.h" +#include "crimson/common/config_proxy.h" + +using Config = crimson::common::ConfigProxy; +const std::string test_uint_option = "osd_max_pgls"; +const uint64_t INVALID_VALUE = (uint64_t)(-1); +const uint64_t EXPECTED_VALUE = 42; + +class ConfigObs : public ceph::md_config_obs_impl<Config> { + uint64_t last_change = INVALID_VALUE; + uint64_t num_changes = 0; + + const char** get_tracked_conf_keys() const override { + static const char* keys[] = { + test_uint_option.c_str(), + nullptr, + }; + return keys; + } + void handle_conf_change(const Config& conf, + const std::set <std::string> &changes) override{ + if (changes.count(test_uint_option)) { + last_change = conf.get_val<uint64_t>(test_uint_option); + num_changes += 1; + } + } +public: + ConfigObs() { + crimson::common::local_conf().add_observer(this); + } + + uint64_t get_last_change() const { return last_change; } + uint64_t get_num_changes() const { return num_changes; } + seastar::future<> stop() { + crimson::common::local_conf().remove_observer(this); + return seastar::make_ready_future<>(); + } +}; + +seastar::sharded<ConfigObs> sharded_cobs; + +static seastar::future<> test_config() +{ + return crimson::common::sharded_conf().start(EntityName{}, string_view{"ceph"}).then([] { + std::vector<const char*> args; + std::string cluster; + std::string conf_file_list; + auto init_params = ceph_argparse_early_args(args, + CEPH_ENTITY_TYPE_CLIENT, + &cluster, + &conf_file_list); + auto& conf = crimson::common::local_conf(); + conf->name = init_params.name; + conf->cluster = cluster; + return conf.parse_config_files(conf_file_list); + }).then([] { + return crimson::common::sharded_conf().invoke_on(0, &Config::start); + }).then([] { + return sharded_cobs.start(); + }).then([] { + auto& conf = crimson::common::local_conf(); + return conf.set_val(test_uint_option, std::to_string(EXPECTED_VALUE)); + }).then([] { + return crimson::common::sharded_conf().invoke_on_all([](Config& config) { + if (config.get_val<uint64_t>(test_uint_option) != EXPECTED_VALUE) { + throw std::runtime_error("configurations don't match"); + } + if (sharded_cobs.local().get_last_change() != EXPECTED_VALUE) { + throw std::runtime_error("last applied changes don't match the latest config"); + } + if (sharded_cobs.local().get_num_changes() != 1) { + throw std::runtime_error("num changes don't match actual changes"); + } + }); + }).finally([] { + return sharded_cobs.stop(); + }).finally([] { + return crimson::common::sharded_conf().stop(); + }); +} + +int main(int argc, char** argv) +{ + seastar::app_template app; + return app.run(argc, argv, [&] { + return test_config().then([] { + std::cout << "All tests succeeded" << std::endl; + }).handle_exception([] (auto eptr) { + std::cout << "Test failure" << std::endl; + return seastar::make_exception_future<>(eptr); + }); + }); +} + + +/* + * Local Variables: + * compile-command: "make -j4 \ + * -C ../../../build \ + * unittest_seastar_config" + * End: + */ diff --git a/src/test/crimson/test_denc.cc b/src/test/crimson/test_denc.cc new file mode 100644 index 000000000..10ebd6dce --- /dev/null +++ b/src/test/crimson/test_denc.cc @@ -0,0 +1,53 @@ +#include <string> +#include <seastar/core/temporary_buffer.hh> +#include <gtest/gtest.h> +#include "include/denc.h" +#include "common/buffer_seastar.h" + +using temporary_buffer = seastar::temporary_buffer<char>; +using buffer_iterator = seastar_buffer_iterator; +using const_buffer_iterator = const_seastar_buffer_iterator; + +template<typename T> +void test_denc(T v) { + // estimate + size_t s = 0; + denc(v, s); + ASSERT_NE(s, 0u); + + // encode + temporary_buffer buf{s}; + buffer_iterator enc{buf}; + denc(v, enc); + size_t len = enc.get() - buf.begin(); + ASSERT_LE(len, s); + + // decode + T out; + temporary_buffer encoded = buf.share(); + encoded.trim(len); + const_buffer_iterator dec{encoded}; + denc(out, dec); + ASSERT_EQ(v, out); + ASSERT_EQ(dec.get(), enc.get()); +} + +TEST(denc, simple) +{ + test_denc((uint8_t)4); + test_denc((int8_t)-5); + test_denc((uint16_t)6); + test_denc((int16_t)-7); + test_denc((uint32_t)8); + test_denc((int32_t)-9); + test_denc((uint64_t)10); + test_denc((int64_t)-11); +} + +TEST(denc, string) +{ + std::string a, b("hi"), c("multi\nline\n"); + test_denc(a); + test_denc(b); + test_denc(c); +} diff --git a/src/test/crimson/test_errorator.cc b/src/test/crimson/test_errorator.cc new file mode 100644 index 000000000..57dbc78cf --- /dev/null +++ b/src/test/crimson/test_errorator.cc @@ -0,0 +1,52 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- +// vim: ts=8 sw=2 smarttab + +#include "test/crimson/gtest_seastar.h" + +#include "crimson/common/errorator.h" +#include "crimson/common/log.h" + +struct errorator_test_t : public seastar_test_suite_t { + using ertr = crimson::errorator<crimson::ct_error::invarg>; + ertr::future<> test_do_until() { + return crimson::do_until([i=0]() mutable { + if (i < 5) { + ++i; + return ertr::make_ready_future<bool>(false); + } else { + return ertr::make_ready_future<bool>(true); + } + }); + } + struct noncopyable_t { + constexpr noncopyable_t() = default; + ~noncopyable_t() = default; + noncopyable_t(noncopyable_t&&) = default; + private: + noncopyable_t(const noncopyable_t&) = delete; + noncopyable_t& operator=(const noncopyable_t&) = delete; + }; + ertr::future<> test_non_copy_then() { + return create_noncopyable().safe_then([](auto t) { + return ertr::now(); + }); + } +private: + ertr::future<noncopyable_t> create_noncopyable() { + return ertr::make_ready_future<noncopyable_t>(); + } +}; + +TEST_F(errorator_test_t, basic) +{ + run_async([this] { + test_do_until().unsafe_get0(); + }); +} + +TEST_F(errorator_test_t, non_copy_then) +{ + run_async([this] { + test_non_copy_then().unsafe_get0(); + }); +} diff --git a/src/test/crimson/test_fixed_kv_node_layout.cc b/src/test/crimson/test_fixed_kv_node_layout.cc new file mode 100644 index 000000000..5f59c4e5e --- /dev/null +++ b/src/test/crimson/test_fixed_kv_node_layout.cc @@ -0,0 +1,376 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include <stdio.h> +#include <iostream> + +#include "gtest/gtest.h" + +#include "crimson/common/fixed_kv_node_layout.h" + +using namespace crimson; +using namespace crimson::common; + +struct test_val_t { + uint32_t t1 = 0; + int32_t t2 = 0; + + bool operator==(const test_val_t &rhs) const { + return rhs.t1 == t1 && rhs.t2 == t2; + } + bool operator!=(const test_val_t &rhs) const { + return !(*this == rhs); + } +}; + +struct test_val_le_t { + ceph_le32 t1 = init_le32(0); + ceph_les32 t2 = init_les32(0); + + test_val_le_t() = default; + test_val_le_t(const test_val_le_t &) = default; + test_val_le_t(const test_val_t &nv) + : t1(init_le32(nv.t1)), t2(init_les32(nv.t2)) {} + + operator test_val_t() const { + return test_val_t{t1, t2}; + } +}; + +struct test_meta_t { + uint32_t t1 = 0; + uint32_t t2 = 0; + + bool operator==(const test_meta_t &rhs) const { + return rhs.t1 == t1 && rhs.t2 == t2; + } + bool operator!=(const test_meta_t &rhs) const { + return !(*this == rhs); + } + + std::pair<test_meta_t, test_meta_t> split_into(uint32_t pivot) const { + return std::make_pair( + test_meta_t{t1, pivot}, + test_meta_t{pivot, t2}); + } + + static test_meta_t merge_from(const test_meta_t &lhs, const test_meta_t &rhs) { + return test_meta_t{lhs.t1, rhs.t2}; + } + + static std::pair<test_meta_t, test_meta_t> + rebalance(const test_meta_t &lhs, const test_meta_t &rhs, uint32_t pivot) { + return std::make_pair( + test_meta_t{lhs.t1, pivot}, + test_meta_t{pivot, rhs.t2}); + } +}; + +struct test_meta_le_t { + ceph_le32 t1 = init_le32(0); + ceph_le32 t2 = init_le32(0); + + test_meta_le_t() = default; + test_meta_le_t(const test_meta_le_t &) = default; + test_meta_le_t(const test_meta_t &nv) + : t1(init_le32(nv.t1)), t2(init_le32(nv.t2)) {} + + operator test_meta_t() const { + return test_meta_t{t1, t2}; + } +}; + +constexpr size_t CAPACITY = 339; + +struct TestNode : FixedKVNodeLayout< + CAPACITY, + test_meta_t, test_meta_le_t, + uint32_t, ceph_le32, + test_val_t, test_val_le_t> { + char buf[4096]; + TestNode() : FixedKVNodeLayout(buf) { + memset(buf, 0, sizeof(buf)); + set_meta({0, std::numeric_limits<uint32_t>::max()}); + } + TestNode(const TestNode &rhs) + : FixedKVNodeLayout(buf) { + ::memcpy(buf, rhs.buf, sizeof(buf)); + } + + TestNode &operator=(const TestNode &rhs) { + memcpy(buf, rhs.buf, sizeof(buf)); + return *this; + } +}; + +TEST(FixedKVNodeTest, basic) { + auto node = TestNode(); + ASSERT_EQ(node.get_size(), 0); + + auto val = test_val_t{ 1, 1 }; + node.journal_insert(node.begin(), 1, val, nullptr); + ASSERT_EQ(node.get_size(), 1); + + auto iter = node.begin(); + ASSERT_EQ(iter.get_key(), 1); + ASSERT_EQ(val, iter.get_val()); + + ASSERT_EQ(std::numeric_limits<uint32_t>::max(), iter.get_next_key_or_max()); +} + +TEST(FixedKVNodeTest, at_capacity) { + auto node = TestNode(); + ASSERT_EQ(CAPACITY, node.get_capacity()); + + ASSERT_EQ(node.get_size(), 0); + + unsigned short num = 0; + auto iter = node.begin(); + while (num < CAPACITY) { + node.journal_insert(iter, num, test_val_t{num, num}, nullptr); + ++num; + ++iter; + } + ASSERT_EQ(node.get_size(), CAPACITY); + + num = 0; + for (auto &i : node) { + ASSERT_EQ(i.get_key(), num); + ASSERT_EQ(i.get_val(), (test_val_t{num, num})); + if (num < (CAPACITY - 1)) { + ASSERT_EQ(i.get_next_key_or_max(), num + 1); + } else { + ASSERT_EQ(std::numeric_limits<uint32_t>::max(), i.get_next_key_or_max()); + } + ++num; + } +} + +TEST(FixedKVNodeTest, split) { + auto node = TestNode(); + + ASSERT_EQ(node.get_size(), 0); + + unsigned short num = 0; + auto iter = node.begin(); + while (num < CAPACITY) { + node.journal_insert(iter, num, test_val_t{num, num}, nullptr); + ++num; + ++iter; + } + ASSERT_EQ(node.get_size(), CAPACITY); + + auto split_left = TestNode(); + auto split_right = TestNode(); + node.split_into(split_left, split_right); + + ASSERT_EQ(split_left.get_size() + split_right.get_size(), CAPACITY); + ASSERT_EQ(split_left.get_meta().t1, split_left.begin()->get_key()); + ASSERT_EQ(split_left.get_meta().t2, split_right.get_meta().t1); + ASSERT_EQ(split_right.get_meta().t2, std::numeric_limits<uint32_t>::max()); + + num = 0; + for (auto &i : split_left) { + ASSERT_EQ(i.get_key(), num); + ASSERT_EQ(i.get_val(), (test_val_t{num, num})); + if (num < split_left.get_size() - 1) { + ASSERT_EQ(i.get_next_key_or_max(), num + 1); + } else { + ASSERT_EQ(std::numeric_limits<uint32_t>::max(), i.get_next_key_or_max()); + } + ++num; + } + for (auto &i : split_right) { + ASSERT_EQ(i.get_key(), num); + ASSERT_EQ(i.get_val(), (test_val_t{num, num})); + if (num < CAPACITY - 1) { + ASSERT_EQ(i.get_next_key_or_max(), num + 1); + } else { + ASSERT_EQ(std::numeric_limits<uint32_t>::max(), i.get_next_key_or_max()); + } + ++num; + } + ASSERT_EQ(num, CAPACITY); +} + +TEST(FixedKVNodeTest, merge) { + auto node = TestNode(); + auto node2 = TestNode(); + + ASSERT_EQ(node.get_size(), 0); + ASSERT_EQ(node2.get_size(), 0); + + unsigned short num = 0; + auto iter = node.begin(); + while (num < CAPACITY/2) { + node.journal_insert(iter, num, test_val_t{num, num}, nullptr); + ++num; + ++iter; + } + node.set_meta({0, num}); + node2.set_meta({num, std::numeric_limits<uint32_t>::max()}); + iter = node2.begin(); + while (num < (2 * (CAPACITY / 2))) { + node2.journal_insert(iter, num, test_val_t{num, num}, nullptr); + ++num; + ++iter; + } + + ASSERT_EQ(node.get_size(), CAPACITY / 2); + ASSERT_EQ(node2.get_size(), CAPACITY / 2); + + auto total = node.get_size() + node2.get_size(); + + auto node_merged = TestNode(); + node_merged.merge_from(node, node2); + + ASSERT_EQ( + node_merged.get_meta(), + (test_meta_t{0, std::numeric_limits<uint32_t>::max()})); + + ASSERT_EQ(node_merged.get_size(), total); + num = 0; + for (auto &i : node_merged) { + ASSERT_EQ(i.get_key(), num); + ASSERT_EQ(i.get_val(), (test_val_t{num, num})); + if (num < node_merged.get_size() - 1) { + ASSERT_EQ(i.get_next_key_or_max(), num + 1); + } else { + ASSERT_EQ(std::numeric_limits<uint32_t>::max(), i.get_next_key_or_max()); + } + ++num; + } + ASSERT_EQ(num, total); +} + +void run_balance_test(unsigned left, unsigned right, bool prefer_left) +{ + auto node = TestNode(); + auto node2 = TestNode(); + + ASSERT_EQ(node.get_size(), 0); + ASSERT_EQ(node2.get_size(), 0); + + unsigned short num = 0; + auto iter = node.begin(); + while (num < left) { + node.journal_insert(iter, num, test_val_t{num, num}, nullptr); + ++num; + ++iter; + } + node.set_meta({0, num}); + node2.set_meta({num, std::numeric_limits<uint32_t>::max()}); + iter = node2.begin(); + while (num < (left + right)) { + node2.journal_insert(iter, num, test_val_t{num, num}, nullptr); + ++num; + ++iter; + } + + ASSERT_EQ(node.get_size(), left); + ASSERT_EQ(node2.get_size(), right); + + auto total = node.get_size() + node2.get_size(); + + auto node_balanced = TestNode(); + auto node_balanced2 = TestNode(); + auto pivot = TestNode::balance_into_new_nodes( + node, + node2, + prefer_left, + node_balanced, + node_balanced2); + + ASSERT_EQ(total, node_balanced.get_size() + node_balanced2.get_size()); + + unsigned left_size, right_size; + if (total % 2) { + if (prefer_left) { + left_size = (total/2) + 1; + right_size = total/2; + } else { + left_size = total/2; + right_size = (total/2) + 1; + } + } else { + left_size = right_size = total/2; + } + ASSERT_EQ(pivot, left_size); + ASSERT_EQ(left_size, node_balanced.get_size()); + ASSERT_EQ(right_size, node_balanced2.get_size()); + + ASSERT_EQ( + node_balanced.get_meta(), + (test_meta_t{0, left_size})); + ASSERT_EQ( + node_balanced2.get_meta(), + (test_meta_t{left_size, std::numeric_limits<uint32_t>::max()})); + + num = 0; + for (auto &i: node_balanced) { + ASSERT_EQ(i.get_key(), num); + ASSERT_EQ(i.get_val(), (test_val_t{num, num})); + if (num < node_balanced.get_size() - 1) { + ASSERT_EQ(i.get_next_key_or_max(), num + 1); + } else { + ASSERT_EQ(std::numeric_limits<uint32_t>::max(), i.get_next_key_or_max()); + } + ++num; + } + for (auto &i: node_balanced2) { + ASSERT_EQ(i.get_key(), num); + ASSERT_EQ(i.get_val(), (test_val_t{num, num})); + if (num < total - 1) { + ASSERT_EQ(i.get_next_key_or_max(), num + 1); + } else { + ASSERT_EQ(std::numeric_limits<uint32_t>::max(), i.get_next_key_or_max()); + } + ++num; + } +} + +TEST(FixedKVNodeTest, balanced) { + run_balance_test(CAPACITY / 2, CAPACITY, true); + run_balance_test(CAPACITY / 2, CAPACITY, false); + run_balance_test(CAPACITY, CAPACITY / 2, true); + run_balance_test(CAPACITY, CAPACITY / 2, false); + run_balance_test(CAPACITY - 1, CAPACITY / 2, true); + run_balance_test(CAPACITY / 2, CAPACITY - 1, false); + run_balance_test(CAPACITY / 2, CAPACITY / 2, false); +} + +void run_replay_test( + std::vector<std::function<void(TestNode&, TestNode::delta_buffer_t&)>> &&f +) { + TestNode node; + for (unsigned i = 0; i < f.size(); ++i) { + TestNode::delta_buffer_t buf; + TestNode replayed = node; + f[i](node, buf); + buf.replay(replayed); + ASSERT_EQ(node.get_size(), replayed.get_size()); + ASSERT_EQ(node, replayed); + } +} + +TEST(FixedKVNodeTest, replay) { + run_replay_test({ + [](auto &n, auto &b) { + n.journal_insert(n.lower_bound(1), 1, test_val_t{1, 1}, &b); + ASSERT_EQ(1, n.get_size()); + }, + [](auto &n, auto &b) { + n.journal_insert(n.lower_bound(3), 3, test_val_t{1, 2}, &b); + ASSERT_EQ(2, n.get_size()); + }, + [](auto &n, auto &b) { + n.journal_remove(n.find(3), &b); + ASSERT_EQ(1, n.get_size()); + }, + [](auto &n, auto &b) { + n.journal_insert(n.lower_bound(2), 2, test_val_t{5, 1}, &b); + ASSERT_EQ(2, n.get_size()); + } + }); + +} diff --git a/src/test/crimson/test_lru.cc b/src/test/crimson/test_lru.cc new file mode 100644 index 000000000..40ab41539 --- /dev/null +++ b/src/test/crimson/test_lru.cc @@ -0,0 +1,213 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com> + * + * Author: Loic Dachary <loic@dachary.org> + * Cheng Cheng <ccheng.leo@gmail.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Library Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Library Public License for more details. + * + */ + +#include <stdio.h> +#include "gtest/gtest.h" +#include "crimson/common/shared_lru.h" + +class LRUTest : public SharedLRU<unsigned int, int> { +public: + auto add(unsigned int key, int value, bool* existed = nullptr) { + auto pv = new int{value}; + auto ptr = insert(key, std::unique_ptr<int>{pv}); + if (existed) { + *existed = (ptr.get() != pv); + } + return ptr; + } +}; + +TEST(LRU, add) { + LRUTest cache; + unsigned int key = 1; + int value1 = 2; + bool existed = false; + { + auto ptr = cache.add(key, value1, &existed); + ASSERT_TRUE(ptr); + ASSERT_TRUE(ptr.get()); + ASSERT_EQ(value1, *ptr); + ASSERT_FALSE(existed); + } + { + auto ptr = cache.add(key, 3, &existed); + ASSERT_EQ(value1, *ptr); + ASSERT_TRUE(existed); + } +} + +TEST(LRU, empty) { + LRUTest cache; + unsigned int key = 1; + bool existed = false; + + ASSERT_TRUE(cache.empty()); + { + int value1 = 2; + auto ptr = cache.add(key, value1, &existed); + ASSERT_EQ(value1, *ptr); + ASSERT_FALSE(existed); + } + ASSERT_FALSE(cache.empty()); + + cache.clear(); + ASSERT_TRUE(cache.empty()); +} + +TEST(LRU, lookup) { + LRUTest cache; + unsigned int key = 1; + { + int value = 2; + auto ptr = cache.add(key, value); + ASSERT_TRUE(ptr); + ASSERT_TRUE(ptr.get()); + ASSERT_TRUE(cache.find(key).get()); + ASSERT_EQ(value, *cache.find(key)); + } + ASSERT_TRUE(cache.find(key).get()); +} + +TEST(LRU, lookup_or_create) { + LRUTest cache; + { + int value = 2; + unsigned int key = 1; + ASSERT_TRUE(cache.add(key, value).get()); + ASSERT_TRUE(cache[key].get()); + ASSERT_EQ(value, *cache.find(key)); + } + { + unsigned int key = 2; + ASSERT_TRUE(cache[key].get()); + ASSERT_EQ(0, *cache.find(key)); + } + ASSERT_TRUE(cache.find(1).get()); + ASSERT_TRUE(cache.find(2).get()); +} + +TEST(LRU, lower_bound) { + LRUTest cache; + + { + unsigned int key = 1; + ASSERT_FALSE(cache.lower_bound(key)); + int value = 2; + + ASSERT_TRUE(cache.add(key, value).get()); + ASSERT_TRUE(cache.lower_bound(key).get()); + EXPECT_EQ(value, *cache.lower_bound(key)); + } +} + +TEST(LRU, get_next) { + + { + LRUTest cache; + const unsigned int key = 0; + EXPECT_FALSE(cache.upper_bound(key)); + } + { + LRUTest cache; + const unsigned int key1 = 111; + auto ptr1 = cache[key1]; + const unsigned int key2 = 222; + auto ptr2 = cache[key2]; + + auto i = cache.upper_bound(0); + ASSERT_TRUE(i); + EXPECT_EQ(i->first, key1); + auto j = cache.upper_bound(i->first); + ASSERT_TRUE(j); + EXPECT_EQ(j->first, key2); + } +} + +TEST(LRU, clear) { + LRUTest cache; + unsigned int key = 1; + int value = 2; + cache.add(key, value); + { + auto found = cache.find(key); + ASSERT_TRUE(found); + ASSERT_EQ(value, *found); + } + ASSERT_TRUE(cache.find(key).get()); + cache.clear(); + ASSERT_FALSE(cache.find(key)); + ASSERT_TRUE(cache.empty()); +} + +TEST(LRU, eviction) { + LRUTest cache{5}; + bool existed; + // add a bunch of elements, some of them will be evicted + for (size_t i = 0; i < 2 * cache.capacity(); ++i) { + cache.add(i, i, &existed); + ASSERT_FALSE(existed); + } + size_t i = 0; + for (; i < cache.capacity(); ++i) { + ASSERT_FALSE(cache.find(i)); + } + for (; i < 2 * cache.capacity(); ++i) { + ASSERT_TRUE(cache.find(i)); + } +} + +TEST(LRU, track_weak) { + constexpr int SIZE = 5; + LRUTest cache{SIZE}; + + bool existed = false; + // strong reference to keep 0 alive + auto ptr = cache.add(0, 0, &existed); + ASSERT_FALSE(existed); + + // add a bunch of elements to get 0 evicted + for (size_t i = 1; i < 2 * cache.capacity(); ++i) { + cache.add(i, i, &existed); + ASSERT_FALSE(existed); + } + // 0 is still reachable via the cache + ASSERT_TRUE(cache.find(0)); + ASSERT_TRUE(cache.find(0).get()); + ASSERT_EQ(0, *cache.find(0)); + + // [0..SIZE) are evicted when adding [SIZE..2*SIZE) + // [SIZE..SIZE * 2) were still in the cache before accessing 0, + // but SIZE got evicted when accessing 0 + ASSERT_FALSE(cache.find(SIZE-1)); + ASSERT_FALSE(cache.find(SIZE)); + ASSERT_TRUE(cache.find(SIZE+1)); + ASSERT_TRUE(cache.find(SIZE+1).get()); + ASSERT_EQ((int)SIZE+1, *cache.find(SIZE+1)); + + ptr.reset(); + // 0 is still reachable, as it is now put back into LRU cache + ASSERT_TRUE(cache.find(0)); +} + +// Local Variables: +// compile-command: "cmake --build ../../../build -j 8 --target unittest_seastar_lru && ctest -R unittest_seastar_lru # --gtest_filter=*.* --log-to-stderr=true" +// End: diff --git a/src/test/crimson/test_messenger.cc b/src/test/crimson/test_messenger.cc new file mode 100644 index 000000000..3877ec570 --- /dev/null +++ b/src/test/crimson/test_messenger.cc @@ -0,0 +1,3668 @@ +#include "common/ceph_argparse.h" +#include "common/ceph_time.h" +#include "messages/MPing.h" +#include "messages/MCommand.h" +#include "messages/MCommandReply.h" +#include "messages/MOSDOp.h" +#include "messages/MOSDOpReply.h" +#include "crimson/auth/DummyAuth.h" +#include "crimson/common/log.h" +#include "crimson/net/Connection.h" +#include "crimson/net/Dispatcher.h" +#include "crimson/net/Messenger.h" +#include "crimson/net/Interceptor.h" + +#include <map> +#include <random> +#include <boost/program_options.hpp> +#include <fmt/format.h> +#include <fmt/ostream.h> +#include <seastar/core/app-template.hh> +#include <seastar/core/do_with.hh> +#include <seastar/core/future-util.hh> +#include <seastar/core/reactor.hh> +#include <seastar/core/sleep.hh> +#include <seastar/core/with_timeout.hh> + +#include "test_cmds.h" + +namespace bpo = boost::program_options; +using crimson::common::local_conf; + +namespace { + +seastar::logger& logger() { + return crimson::get_logger(ceph_subsys_ms); +} + +static std::random_device rd; +static std::default_random_engine rng{rd()}; +static bool verbose = false; + +static entity_addr_t get_server_addr() { + static int port = 9030; + ++port; + entity_addr_t saddr; + saddr.parse("127.0.0.1", nullptr); + saddr.set_port(port); + return saddr; +} + +static seastar::future<> test_echo(unsigned rounds, + double keepalive_ratio, + bool v2) +{ + struct test_state { + struct Server final + : public crimson::net::Dispatcher { + crimson::net::MessengerRef msgr; + crimson::auth::DummyAuthClientServer dummy_auth; + + std::optional<seastar::future<>> ms_dispatch( + crimson::net::ConnectionRef c, MessageRef m) override { + if (verbose) { + logger().info("server got {}", *m); + } + // reply with a pong + std::ignore = c->send(make_message<MPing>()); + return {seastar::now()}; + } + + seastar::future<> init(const entity_name_t& name, + const std::string& lname, + const uint64_t nonce, + const entity_addr_t& addr) { + msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); + msgr->set_require_authorizer(false); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + return msgr->bind(entity_addrvec_t{addr}).safe_then([this] { + return msgr->start({this}); + }, crimson::net::Messenger::bind_ertr::all_same_way( + [addr] (const std::error_code& e) { + logger().error("test_echo(): " + "there is another instance running at {}", addr); + ceph_abort(); + })); + } + seastar::future<> shutdown() { + ceph_assert(msgr); + msgr->stop(); + return msgr->shutdown(); + } + }; + + struct Client final + : public crimson::net::Dispatcher { + struct PingSession : public seastar::enable_shared_from_this<PingSession> { + unsigned count = 0u; + mono_time connected_time; + mono_time finish_time; + }; + using PingSessionRef = seastar::shared_ptr<PingSession>; + + unsigned rounds; + std::bernoulli_distribution keepalive_dist; + crimson::net::MessengerRef msgr; + std::map<crimson::net::ConnectionRef, seastar::promise<>> pending_conns; + std::map<crimson::net::ConnectionRef, PingSessionRef> sessions; + crimson::auth::DummyAuthClientServer dummy_auth; + + Client(unsigned rounds, double keepalive_ratio) + : rounds(rounds), + keepalive_dist(std::bernoulli_distribution{keepalive_ratio}) {} + + PingSessionRef find_session(crimson::net::ConnectionRef c) { + auto found = sessions.find(c); + if (found == sessions.end()) { + ceph_assert(false); + } + return found->second; + } + + void ms_handle_connect(crimson::net::ConnectionRef conn) override { + auto session = seastar::make_shared<PingSession>(); + auto [i, added] = sessions.emplace(conn, session); + std::ignore = i; + ceph_assert(added); + session->connected_time = mono_clock::now(); + } + std::optional<seastar::future<>> ms_dispatch( + crimson::net::ConnectionRef c, MessageRef m) override { + auto session = find_session(c); + ++(session->count); + if (verbose) { + logger().info("client ms_dispatch {}", session->count); + } + + if (session->count == rounds) { + logger().info("{}: finished receiving {} pongs", *c, session->count); + session->finish_time = mono_clock::now(); + auto found = pending_conns.find(c); + ceph_assert(found != pending_conns.end()); + found->second.set_value(); + } + return {seastar::now()}; + } + + seastar::future<> init(const entity_name_t& name, + const std::string& lname, + const uint64_t nonce) { + msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + return msgr->start({this}); + } + + seastar::future<> shutdown() { + ceph_assert(msgr); + msgr->stop(); + return msgr->shutdown(); + } + + seastar::future<> dispatch_pingpong(const entity_addr_t& peer_addr) { + mono_time start_time = mono_clock::now(); + auto conn = msgr->connect(peer_addr, entity_name_t::TYPE_OSD); + return seastar::futurize_invoke([this, conn] { + return do_dispatch_pingpong(conn); + }).then([this, conn, start_time] { + auto session = find_session(conn); + std::chrono::duration<double> dur_handshake = session->connected_time - start_time; + std::chrono::duration<double> dur_pingpong = session->finish_time - session->connected_time; + logger().info("{}: handshake {}, pingpong {}", + *conn, dur_handshake.count(), dur_pingpong.count()); + }); + } + + private: + seastar::future<> do_dispatch_pingpong(crimson::net::ConnectionRef conn) { + auto [i, added] = pending_conns.emplace(conn, seastar::promise<>()); + std::ignore = i; + ceph_assert(added); + return seastar::do_with(0u, 0u, + [this, conn](auto &count_ping, auto &count_keepalive) { + return seastar::do_until( + [this, conn, &count_ping, &count_keepalive] { + bool stop = (count_ping == rounds); + if (stop) { + logger().info("{}: finished sending {} pings with {} keepalives", + *conn, count_ping, count_keepalive); + } + return stop; + }, + [this, conn, &count_ping, &count_keepalive] { + return seastar::repeat([this, conn, &count_ping, &count_keepalive] { + if (keepalive_dist(rng)) { + return conn->keepalive() + .then([&count_keepalive] { + count_keepalive += 1; + return seastar::make_ready_future<seastar::stop_iteration>( + seastar::stop_iteration::no); + }); + } else { + return conn->send(make_message<MPing>()) + .then([&count_ping] { + count_ping += 1; + return seastar::make_ready_future<seastar::stop_iteration>( + seastar::stop_iteration::yes); + }); + } + }); + }).then([this, conn] { + auto found = pending_conns.find(conn); + return found->second.get_future(); + } + ); + }); + } + }; + }; + + logger().info("test_echo(rounds={}, keepalive_ratio={}, v2={}):", + rounds, keepalive_ratio, v2); + auto server1 = seastar::make_shared<test_state::Server>(); + auto server2 = seastar::make_shared<test_state::Server>(); + auto client1 = seastar::make_shared<test_state::Client>(rounds, keepalive_ratio); + auto client2 = seastar::make_shared<test_state::Client>(rounds, keepalive_ratio); + // start servers and clients + auto addr1 = get_server_addr(); + auto addr2 = get_server_addr(); + if (v2) { + addr1.set_type(entity_addr_t::TYPE_MSGR2); + addr2.set_type(entity_addr_t::TYPE_MSGR2); + } else { + addr1.set_type(entity_addr_t::TYPE_LEGACY); + addr2.set_type(entity_addr_t::TYPE_LEGACY); + } + return seastar::when_all_succeed( + server1->init(entity_name_t::OSD(0), "server1", 1, addr1), + server2->init(entity_name_t::OSD(1), "server2", 2, addr2), + client1->init(entity_name_t::OSD(2), "client1", 3), + client2->init(entity_name_t::OSD(3), "client2", 4) + // dispatch pingpoing + ).then_unpack([client1, client2, server1, server2] { + return seastar::when_all_succeed( + // test connecting in parallel, accepting in parallel + client1->dispatch_pingpong(server2->msgr->get_myaddr()), + client2->dispatch_pingpong(server1->msgr->get_myaddr())); + // shutdown + }).then_unpack([] { + return seastar::now(); + }).then([client1] { + logger().info("client1 shutdown..."); + return client1->shutdown(); + }).then([client2] { + logger().info("client2 shutdown..."); + return client2->shutdown(); + }).then([server1] { + logger().info("server1 shutdown..."); + return server1->shutdown(); + }).then([server2] { + logger().info("server2 shutdown..."); + return server2->shutdown(); + }).then([] { + logger().info("test_echo() done!\n"); + }).handle_exception([server1, server2, client1, client2] (auto eptr) { + logger().error("test_echo() failed: got exception {}", eptr); + throw; + }); +} + +static seastar::future<> test_concurrent_dispatch(bool v2) +{ + struct test_state { + struct Server final + : public crimson::net::Dispatcher { + crimson::net::MessengerRef msgr; + int count = 0; + seastar::promise<> on_second; // satisfied on second dispatch + seastar::promise<> on_done; // satisfied when first dispatch unblocks + crimson::auth::DummyAuthClientServer dummy_auth; + + std::optional<seastar::future<>> ms_dispatch( + crimson::net::ConnectionRef, MessageRef m) override { + switch (++count) { + case 1: + // block on the first request until we reenter with the second + std::ignore = on_second.get_future().then([this] { on_done.set_value(); }); + break; + case 2: + on_second.set_value(); + break; + default: + throw std::runtime_error("unexpected count"); + } + return {seastar::now()}; + } + + seastar::future<> wait() { return on_done.get_future(); } + + seastar::future<> init(const entity_name_t& name, + const std::string& lname, + const uint64_t nonce, + const entity_addr_t& addr) { + msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + return msgr->bind(entity_addrvec_t{addr}).safe_then([this] { + return msgr->start({this}); + }, crimson::net::Messenger::bind_ertr::all_same_way( + [addr] (const std::error_code& e) { + logger().error("test_concurrent_dispatch(): " + "there is another instance running at {}", addr); + ceph_abort(); + })); + } + }; + + struct Client final + : public crimson::net::Dispatcher { + crimson::net::MessengerRef msgr; + crimson::auth::DummyAuthClientServer dummy_auth; + + std::optional<seastar::future<>> ms_dispatch( + crimson::net::ConnectionRef, MessageRef m) override { + return {seastar::now()}; + } + + seastar::future<> init(const entity_name_t& name, + const std::string& lname, + const uint64_t nonce) { + msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + return msgr->start({this}); + } + }; + }; + + logger().info("test_concurrent_dispatch(v2={}):", v2); + auto server = seastar::make_shared<test_state::Server>(); + auto client = seastar::make_shared<test_state::Client>(); + auto addr = get_server_addr(); + if (v2) { + addr.set_type(entity_addr_t::TYPE_MSGR2); + } else { + addr.set_type(entity_addr_t::TYPE_LEGACY); + } + addr.set_family(AF_INET); + return seastar::when_all_succeed( + server->init(entity_name_t::OSD(4), "server3", 5, addr), + client->init(entity_name_t::OSD(5), "client3", 6) + ).then_unpack([server, client] { + auto conn = client->msgr->connect(server->msgr->get_myaddr(), + entity_name_t::TYPE_OSD); + // send two messages + return conn->send(make_message<MPing>()).then([conn] { + return conn->send(make_message<MPing>()); + }); + }).then([server] { + return server->wait(); + }).then([client] { + logger().info("client shutdown..."); + client->msgr->stop(); + return client->msgr->shutdown(); + }).then([server] { + logger().info("server shutdown..."); + server->msgr->stop(); + return server->msgr->shutdown(); + }).then([] { + logger().info("test_concurrent_dispatch() done!\n"); + }).handle_exception([server, client] (auto eptr) { + logger().error("test_concurrent_dispatch() failed: got exception {}", eptr); + throw; + }); +} + +seastar::future<> test_preemptive_shutdown(bool v2) { + struct test_state { + class Server final + : public crimson::net::Dispatcher { + crimson::net::MessengerRef msgr; + crimson::auth::DummyAuthClientServer dummy_auth; + + std::optional<seastar::future<>> ms_dispatch( + crimson::net::ConnectionRef c, MessageRef m) override { + std::ignore = c->send(make_message<MPing>()); + return {seastar::now()}; + } + + public: + seastar::future<> init(const entity_name_t& name, + const std::string& lname, + const uint64_t nonce, + const entity_addr_t& addr) { + msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr->set_default_policy(crimson::net::SocketPolicy::stateless_server(0)); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + return msgr->bind(entity_addrvec_t{addr}).safe_then([this] { + return msgr->start({this}); + }, crimson::net::Messenger::bind_ertr::all_same_way( + [addr] (const std::error_code& e) { + logger().error("test_preemptive_shutdown(): " + "there is another instance running at {}", addr); + ceph_abort(); + })); + } + entity_addr_t get_addr() const { + return msgr->get_myaddr(); + } + seastar::future<> shutdown() { + msgr->stop(); + return msgr->shutdown(); + } + }; + + class Client final + : public crimson::net::Dispatcher { + crimson::net::MessengerRef msgr; + crimson::auth::DummyAuthClientServer dummy_auth; + + bool stop_send = false; + seastar::promise<> stopped_send_promise; + + std::optional<seastar::future<>> ms_dispatch( + crimson::net::ConnectionRef, MessageRef m) override { + return {seastar::now()}; + } + + public: + seastar::future<> init(const entity_name_t& name, + const std::string& lname, + const uint64_t nonce) { + msgr = crimson::net::Messenger::create(name, lname, nonce); + msgr->set_default_policy(crimson::net::SocketPolicy::lossy_client(0)); + msgr->set_auth_client(&dummy_auth); + msgr->set_auth_server(&dummy_auth); + return msgr->start({this}); + } + void send_pings(const entity_addr_t& addr) { + auto conn = msgr->connect(addr, entity_name_t::TYPE_OSD); + // forwarded to stopped_send_promise + (void) seastar::do_until( + [this] { return stop_send; }, + [conn] { + return conn->send(make_message<MPing>()).then([] { + return seastar::sleep(0ms); + }); + } + ).then_wrapped([this, conn] (auto fut) { + fut.forward_to(std::move(stopped_send_promise)); + }); + } + seastar::future<> shutdown() { + msgr->stop(); + return msgr->shutdown().then([this] { + stop_send = true; + return stopped_send_promise.get_future(); + }); + } + }; + }; + + logger().info("test_preemptive_shutdown(v2={}):", v2); + auto server = seastar::make_shared<test_state::Server>(); + auto client = seastar::make_shared<test_state::Client>(); + auto addr = get_server_addr(); + if (v2) { + addr.set_type(entity_addr_t::TYPE_MSGR2); + } else { + addr.set_type(entity_addr_t::TYPE_LEGACY); + } + addr.set_family(AF_INET); + return seastar::when_all_succeed( + server->init(entity_name_t::OSD(6), "server4", 7, addr), + client->init(entity_name_t::OSD(7), "client4", 8) + ).then_unpack([server, client] { + client->send_pings(server->get_addr()); + return seastar::sleep(100ms); + }).then([client] { + logger().info("client shutdown..."); + return client->shutdown(); + }).then([server] { + logger().info("server shutdown..."); + return server->shutdown(); + }).then([] { + logger().info("test_preemptive_shutdown() done!\n"); + }).handle_exception([server, client] (auto eptr) { + logger().error("test_preemptive_shutdown() failed: got exception {}", eptr); + throw; + }); +} + +using ceph::msgr::v2::Tag; +using crimson::net::bp_action_t; +using crimson::net::bp_type_t; +using crimson::net::Breakpoint; +using crimson::net::Connection; +using crimson::net::ConnectionRef; +using crimson::net::custom_bp_t; +using crimson::net::Dispatcher; +using crimson::net::Interceptor; +using crimson::net::Messenger; +using crimson::net::MessengerRef; +using crimson::net::SocketPolicy; +using crimson::net::tag_bp_t; +using ceph::net::test::cmd_t; +using ceph::net::test::policy_t; + +struct counter_t { unsigned counter = 0; }; + +enum class conn_state_t { + unknown = 0, + established, + closed, + replaced, +}; + +std::ostream& operator<<(std::ostream& out, const conn_state_t& state) { + switch(state) { + case conn_state_t::unknown: + return out << "unknown"; + case conn_state_t::established: + return out << "established"; + case conn_state_t::closed: + return out << "closed"; + case conn_state_t::replaced: + return out << "replaced"; + default: + ceph_abort(); + } +} + +struct ConnResult { + ConnectionRef conn; + unsigned index; + conn_state_t state = conn_state_t::unknown; + + unsigned connect_attempts = 0; + unsigned client_connect_attempts = 0; + unsigned client_reconnect_attempts = 0; + unsigned cnt_connect_dispatched = 0; + + unsigned accept_attempts = 0; + unsigned server_connect_attempts = 0; + unsigned server_reconnect_attempts = 0; + unsigned cnt_accept_dispatched = 0; + + unsigned cnt_reset_dispatched = 0; + unsigned cnt_remote_reset_dispatched = 0; + + ConnResult(Connection& conn, unsigned index) + : conn(conn.shared_from_this()), index(index) {} + + template <typename T> + void _assert_eq(const char* expr_actual, T actual, + const char* expr_expected, T expected) const { + if (actual != expected) { + throw std::runtime_error(fmt::format( + "[{}] {} '{}' is actually {}, not the expected '{}' {}", + index, *conn, expr_actual, actual, expr_expected, expected)); + } + } + +#define ASSERT_EQUAL(actual, expected) \ + _assert_eq(#actual, actual, #expected, expected) + + void assert_state_at(conn_state_t expected) const { + ASSERT_EQUAL(state, expected); + } + + void assert_connect(unsigned attempts, + unsigned connects, + unsigned reconnects, + unsigned dispatched) const { + ASSERT_EQUAL(connect_attempts, attempts); + ASSERT_EQUAL(client_connect_attempts, connects); + ASSERT_EQUAL(client_reconnect_attempts, reconnects); + ASSERT_EQUAL(cnt_connect_dispatched, dispatched); + } + + void assert_connect(unsigned attempts, + unsigned dispatched) const { + ASSERT_EQUAL(connect_attempts, attempts); + ASSERT_EQUAL(cnt_connect_dispatched, dispatched); + } + + void assert_accept(unsigned attempts, + unsigned accepts, + unsigned reaccepts, + unsigned dispatched) const { + ASSERT_EQUAL(accept_attempts, attempts); + ASSERT_EQUAL(server_connect_attempts, accepts); + ASSERT_EQUAL(server_reconnect_attempts, reaccepts); + ASSERT_EQUAL(cnt_accept_dispatched, dispatched); + } + + void assert_accept(unsigned attempts, + unsigned dispatched) const { + ASSERT_EQUAL(accept_attempts, attempts); + ASSERT_EQUAL(cnt_accept_dispatched, dispatched); + } + + void assert_reset(unsigned local, unsigned remote) const { + ASSERT_EQUAL(cnt_reset_dispatched, local); + ASSERT_EQUAL(cnt_remote_reset_dispatched, remote); + } + + void dump() const { + logger().info("\nResult({}):\n" + " conn: [{}] {}:\n" + " state: {}\n" + " connect_attempts: {}\n" + " client_connect_attempts: {}\n" + " client_reconnect_attempts: {}\n" + " cnt_connect_dispatched: {}\n" + " accept_attempts: {}\n" + " server_connect_attempts: {}\n" + " server_reconnect_attempts: {}\n" + " cnt_accept_dispatched: {}\n" + " cnt_reset_dispatched: {}\n" + " cnt_remote_reset_dispatched: {}\n", + static_cast<const void*>(this), + index, *conn, + state, + connect_attempts, + client_connect_attempts, + client_reconnect_attempts, + cnt_connect_dispatched, + accept_attempts, + server_connect_attempts, + server_reconnect_attempts, + cnt_accept_dispatched, + cnt_reset_dispatched, + cnt_remote_reset_dispatched); + } +}; +using ConnResults = std::vector<ConnResult>; + +struct TestInterceptor : public Interceptor { + std::map<Breakpoint, std::map<unsigned, bp_action_t>> breakpoints; + std::map<Breakpoint, counter_t> breakpoints_counter; + std::map<ConnectionRef, unsigned> conns; + ConnResults results; + std::optional<seastar::abort_source> signal; + + TestInterceptor() = default; + // only used for copy breakpoint configurations + TestInterceptor(const TestInterceptor& other) { + assert(other.breakpoints_counter.empty()); + assert(other.conns.empty()); + assert(other.results.empty()); + breakpoints = other.breakpoints; + assert(!other.signal); + } + + void make_fault(Breakpoint bp, unsigned round = 1) { + assert(round >= 1); + breakpoints[bp][round] = bp_action_t::FAULT; + } + + void make_block(Breakpoint bp, unsigned round = 1) { + assert(round >= 1); + breakpoints[bp][round] = bp_action_t::BLOCK; + } + + void make_stall(Breakpoint bp, unsigned round = 1) { + assert(round >= 1); + breakpoints[bp][round] = bp_action_t::STALL; + } + + ConnResult* find_result(ConnectionRef conn) { + auto it = conns.find(conn); + if (it == conns.end()) { + return nullptr; + } else { + return &results[it->second]; + } + } + + seastar::future<> wait() { + assert(!signal); + signal = seastar::abort_source(); + return seastar::sleep_abortable(10s, *signal).then([] { + throw std::runtime_error("Timeout (10s) in TestInterceptor::wait()"); + }).handle_exception_type([] (const seastar::sleep_aborted& e) { + // wait done! + }); + } + + void notify() { + if (signal) { + signal->request_abort(); + signal = std::nullopt; + } + } + + private: + void register_conn(Connection& conn) override { + unsigned index = results.size(); + results.emplace_back(conn, index); + conns[conn.shared_from_this()] = index; + notify(); + logger().info("[{}] {} new connection registered", index, conn); + } + + void register_conn_closed(Connection& conn) override { + auto result = find_result(conn.shared_from_this()); + if (result == nullptr) { + logger().error("Untracked closed connection: {}", conn); + ceph_abort(); + } + + if (result->state != conn_state_t::replaced) { + result->state = conn_state_t::closed; + } + notify(); + logger().info("[{}] {} closed({})", result->index, conn, result->state); + } + + void register_conn_ready(Connection& conn) override { + auto result = find_result(conn.shared_from_this()); + if (result == nullptr) { + logger().error("Untracked ready connection: {}", conn); + ceph_abort(); + } + + ceph_assert(conn.is_connected()); + notify(); + logger().info("[{}] {} ready", result->index, conn); + } + + void register_conn_replaced(Connection& conn) override { + auto result = find_result(conn.shared_from_this()); + if (result == nullptr) { + logger().error("Untracked replaced connection: {}", conn); + ceph_abort(); + } + + result->state = conn_state_t::replaced; + logger().info("[{}] {} {}", result->index, conn, result->state); + } + + bp_action_t intercept(Connection& conn, Breakpoint bp) override { + ++breakpoints_counter[bp].counter; + + auto result = find_result(conn.shared_from_this()); + if (result == nullptr) { + logger().error("Untracked intercepted connection: {}, at breakpoint {}({})", + conn, bp, breakpoints_counter[bp].counter); + ceph_abort(); + } + + if (bp == custom_bp_t::SOCKET_CONNECTING) { + ++result->connect_attempts; + logger().info("[Test] connect_attempts={}", result->connect_attempts); + } else if (bp == tag_bp_t{Tag::CLIENT_IDENT, bp_type_t::WRITE}) { + ++result->client_connect_attempts; + logger().info("[Test] client_connect_attempts={}", result->client_connect_attempts); + } else if (bp == tag_bp_t{Tag::SESSION_RECONNECT, bp_type_t::WRITE}) { + ++result->client_reconnect_attempts; + logger().info("[Test] client_reconnect_attempts={}", result->client_reconnect_attempts); + } else if (bp == custom_bp_t::SOCKET_ACCEPTED) { + ++result->accept_attempts; + logger().info("[Test] accept_attempts={}", result->accept_attempts); + } else if (bp == tag_bp_t{Tag::CLIENT_IDENT, bp_type_t::READ}) { + ++result->server_connect_attempts; + logger().info("[Test] server_connect_attemps={}", result->server_connect_attempts); + } else if (bp == tag_bp_t{Tag::SESSION_RECONNECT, bp_type_t::READ}) { + ++result->server_reconnect_attempts; + logger().info("[Test] server_reconnect_attempts={}", result->server_reconnect_attempts); + } + + auto it_bp = breakpoints.find(bp); + if (it_bp != breakpoints.end()) { + auto it_cnt = it_bp->second.find(breakpoints_counter[bp].counter); + if (it_cnt != it_bp->second.end()) { + logger().info("[{}] {} intercepted {}({}) => {}", + result->index, conn, bp, + breakpoints_counter[bp].counter, it_cnt->second); + return it_cnt->second; + } + } + logger().info("[{}] {} intercepted {}({})", + result->index, conn, bp, breakpoints_counter[bp].counter); + return bp_action_t::CONTINUE; + } +}; + +SocketPolicy to_socket_policy(policy_t policy) { + switch (policy) { + case policy_t::stateful_server: + return SocketPolicy::stateful_server(0); + case policy_t::stateless_server: + return SocketPolicy::stateless_server(0); + case policy_t::lossless_peer: + return SocketPolicy::lossless_peer(0); + case policy_t::lossless_peer_reuse: + return SocketPolicy::lossless_peer_reuse(0); + case policy_t::lossy_client: + return SocketPolicy::lossy_client(0); + case policy_t::lossless_client: + return SocketPolicy::lossless_client(0); + default: + logger().error("unexpected policy type"); + ceph_abort(); + } +} + +class FailoverSuite : public Dispatcher { + crimson::auth::DummyAuthClientServer dummy_auth; + MessengerRef test_msgr; + const entity_addr_t test_peer_addr; + TestInterceptor interceptor; + + unsigned tracked_index = 0; + ConnectionRef tracked_conn; + unsigned pending_send = 0; + unsigned pending_peer_receive = 0; + unsigned pending_receive = 0; + + std::optional<seastar::future<>> ms_dispatch(ConnectionRef c, MessageRef m) override { + auto result = interceptor.find_result(c); + if (result == nullptr) { + logger().error("Untracked ms dispatched connection: {}", *c); + ceph_abort(); + } + + if (tracked_conn != c) { + logger().error("[{}] {} got op, but doesn't match tracked_conn [{}] {}", + result->index, *c, tracked_index, *tracked_conn); + ceph_abort(); + } + ceph_assert(result->index == tracked_index); + + ceph_assert(m->get_type() == CEPH_MSG_OSD_OP); + ceph_assert(pending_receive > 0); + --pending_receive; + if (pending_receive == 0) { + interceptor.notify(); + } + logger().info("[Test] got op, left {} ops -- [{}] {}", + pending_receive, result->index, *c); + return {seastar::now()}; + } + + void ms_handle_accept(ConnectionRef conn) override { + auto result = interceptor.find_result(conn); + if (result == nullptr) { + logger().error("Untracked accepted connection: {}", *conn); + ceph_abort(); + } + + if (tracked_conn && + !tracked_conn->is_closed() && + tracked_conn != conn) { + logger().error("[{}] {} got accepted, but there's already traced_conn [{}] {}", + result->index, *conn, tracked_index, *tracked_conn); + ceph_abort(); + } + + tracked_index = result->index; + tracked_conn = conn; + ++result->cnt_accept_dispatched; + logger().info("[Test] got accept (cnt_accept_dispatched={}), track [{}] {}", + result->cnt_accept_dispatched, result->index, *conn); + std::ignore = flush_pending_send(); + } + + void ms_handle_connect(ConnectionRef conn) override { + auto result = interceptor.find_result(conn); + if (result == nullptr) { + logger().error("Untracked connected connection: {}", *conn); + ceph_abort(); + } + + if (tracked_conn != conn) { + logger().error("[{}] {} got connected, but doesn't match tracked_conn [{}] {}", + result->index, *conn, tracked_index, *tracked_conn); + ceph_abort(); + } + ceph_assert(result->index == tracked_index); + + ++result->cnt_connect_dispatched; + logger().info("[Test] got connected (cnt_connect_dispatched={}) -- [{}] {}", + result->cnt_connect_dispatched, result->index, *conn); + } + + void ms_handle_reset(ConnectionRef conn, bool is_replace) override { + auto result = interceptor.find_result(conn); + if (result == nullptr) { + logger().error("Untracked reset connection: {}", *conn); + ceph_abort(); + } + + if (tracked_conn != conn) { + logger().error("[{}] {} got reset, but doesn't match tracked_conn [{}] {}", + result->index, *conn, tracked_index, *tracked_conn); + ceph_abort(); + } + ceph_assert(result->index == tracked_index); + + tracked_index = 0; + tracked_conn = nullptr; + ++result->cnt_reset_dispatched; + logger().info("[Test] got reset (cnt_reset_dispatched={}), untrack [{}] {}", + result->cnt_reset_dispatched, result->index, *conn); + } + + void ms_handle_remote_reset(ConnectionRef conn) override { + auto result = interceptor.find_result(conn); + if (result == nullptr) { + logger().error("Untracked remotely reset connection: {}", *conn); + ceph_abort(); + } + + if (tracked_conn != conn) { + logger().error("[{}] {} got remotely reset, but doesn't match tracked_conn [{}] {}", + result->index, *conn, tracked_index, *tracked_conn); + ceph_abort(); + } + ceph_assert(result->index == tracked_index); + + ++result->cnt_remote_reset_dispatched; + logger().info("[Test] got remote reset (cnt_remote_reset_dispatched={}) -- [{}] {}", + result->cnt_remote_reset_dispatched, result->index, *conn); + } + + private: + seastar::future<> init(entity_addr_t addr, SocketPolicy policy) { + test_msgr->set_default_policy(policy); + test_msgr->set_auth_client(&dummy_auth); + test_msgr->set_auth_server(&dummy_auth); + test_msgr->interceptor = &interceptor; + return test_msgr->bind(entity_addrvec_t{addr}).safe_then([this] { + return test_msgr->start({this}); + }, Messenger::bind_ertr::all_same_way([addr] (const std::error_code& e) { + logger().error("FailoverSuite: " + "there is another instance running at {}", addr); + ceph_abort(); + })); + } + + seastar::future<> send_op(bool expect_reply=true) { + ceph_assert(tracked_conn); + if (expect_reply) { + ++pending_peer_receive; + } + pg_t pgid; + object_locator_t oloc; + hobject_t hobj(object_t(), oloc.key, CEPH_NOSNAP, pgid.ps(), + pgid.pool(), oloc.nspace); + spg_t spgid(pgid); + return tracked_conn->send(make_message<MOSDOp>(0, 0, hobj, spgid, 0, 0, 0)); + } + + seastar::future<> flush_pending_send() { + if (pending_send != 0) { + logger().info("[Test] flush sending {} ops", pending_send); + } + ceph_assert(tracked_conn); + return seastar::do_until( + [this] { return pending_send == 0; }, + [this] { + --pending_send; + return send_op(); + }); + } + + seastar::future<> wait_ready(unsigned num_ready_conns, + unsigned num_replaced, + bool wait_received) { + unsigned pending_conns = 0; + unsigned pending_establish = 0; + unsigned replaced_conns = 0; + for (auto& result : interceptor.results) { + if (result.conn->is_closed_clean()) { + if (result.state == conn_state_t::replaced) { + ++replaced_conns; + } + } else if (result.conn->is_connected()) { + if (tracked_conn != result.conn || tracked_index != result.index) { + throw std::runtime_error(fmt::format( + "The connected connection [{}] {} doesn't" + " match the tracked connection [{}] {}", + result.index, *result.conn, tracked_index, tracked_conn)); + } + if (pending_send == 0 && pending_peer_receive == 0 && pending_receive == 0) { + result.state = conn_state_t::established; + } else { + ++pending_establish; + } + } else { + ++pending_conns; + } + } + + bool do_wait = false; + if (num_ready_conns > 0) { + if (interceptor.results.size() > num_ready_conns) { + throw std::runtime_error(fmt::format( + "{} connections, more than expected: {}", + interceptor.results.size(), num_ready_conns)); + } else if (interceptor.results.size() < num_ready_conns || pending_conns > 0) { + logger().info("[Test] wait_ready(): wait for connections," + " currently {} out of {}, pending {} ready ...", + interceptor.results.size(), num_ready_conns, pending_conns); + do_wait = true; + } + } + if (wait_received && + (pending_send || pending_peer_receive || pending_receive)) { + if (pending_conns || pending_establish) { + logger().info("[Test] wait_ready(): wait for pending_send={}," + " pending_peer_receive={}, pending_receive={}," + " pending {}/{} ready/establish connections ...", + pending_send, pending_peer_receive, pending_receive, + pending_conns, pending_establish); + do_wait = true; + } + } + if (num_replaced > 0) { + if (replaced_conns > num_replaced) { + throw std::runtime_error(fmt::format( + "{} replaced connections, more than expected: {}", + replaced_conns, num_replaced)); + } + if (replaced_conns < num_replaced) { + logger().info("[Test] wait_ready(): wait for {} replaced connections," + " currently {} ...", + num_replaced, replaced_conns); + do_wait = true; + } + } + + if (do_wait) { + return interceptor.wait( + ).then([this, num_ready_conns, num_replaced, wait_received] { + return wait_ready(num_ready_conns, num_replaced, wait_received); + }); + } else { + logger().info("[Test] wait_ready(): wait done!"); + return seastar::now(); + } + } + + // called by FailoverTest + public: + FailoverSuite(MessengerRef test_msgr, + entity_addr_t test_peer_addr, + const TestInterceptor& interceptor) + : test_msgr(test_msgr), + test_peer_addr(test_peer_addr), + interceptor(interceptor) { } + + entity_addr_t get_addr() const { + return test_msgr->get_myaddr(); + } + + seastar::future<> shutdown() { + test_msgr->stop(); + return test_msgr->shutdown(); + } + + void needs_receive() { + ++pending_receive; + } + + void notify_peer_reply() { + ceph_assert(pending_peer_receive > 0); + --pending_peer_receive; + logger().info("[Test] TestPeer said got op, left {} ops", + pending_peer_receive); + if (pending_peer_receive == 0) { + interceptor.notify(); + } + } + + void post_check() const { + // make sure all breakpoints were hit + for (auto& kv : interceptor.breakpoints) { + auto it = interceptor.breakpoints_counter.find(kv.first); + if (it == interceptor.breakpoints_counter.end()) { + throw std::runtime_error(fmt::format("{} was missed", kv.first)); + } + auto expected = kv.second.rbegin()->first; + if (expected > it->second.counter) { + throw std::runtime_error(fmt::format( + "{} only triggered {} times, not the expected {}", + kv.first, it->second.counter, expected)); + } + } + } + + void dump_results() const { + for (auto& result : interceptor.results) { + result.dump(); + } + } + + static seastar::future<std::unique_ptr<FailoverSuite>> + create(entity_addr_t test_addr, + SocketPolicy test_policy, + entity_addr_t test_peer_addr, + const TestInterceptor& interceptor) { + auto suite = std::make_unique<FailoverSuite>( + Messenger::create(entity_name_t::OSD(2), "Test", 2), + test_peer_addr, interceptor); + return suite->init(test_addr, test_policy + ).then([suite = std::move(suite)] () mutable { + return std::move(suite); + }); + } + + // called by tests + public: + seastar::future<> connect_peer() { + logger().info("[Test] connect_peer({})", test_peer_addr); + auto conn = test_msgr->connect(test_peer_addr, entity_name_t::TYPE_OSD); + auto result = interceptor.find_result(conn); + ceph_assert(result != nullptr); + + if (tracked_conn) { + if (tracked_conn->is_closed()) { + ceph_assert(tracked_conn != conn); + logger().info("[Test] this is a new session replacing an closed one"); + } else { + ceph_assert(tracked_index == result->index); + ceph_assert(tracked_conn == conn); + logger().info("[Test] this is not a new session"); + } + } else { + logger().info("[Test] this is a new session"); + } + tracked_index = result->index; + tracked_conn = conn; + + return flush_pending_send(); + } + + seastar::future<> send_peer() { + if (tracked_conn) { + logger().info("[Test] send_peer()"); + ceph_assert(!pending_send); + return send_op(); + } else { + ++pending_send; + logger().info("[Test] send_peer() (pending {})", pending_send); + return seastar::now(); + } + } + + seastar::future<> keepalive_peer() { + logger().info("[Test] keepalive_peer()"); + ceph_assert(tracked_conn); + return tracked_conn->keepalive(); + } + + seastar::future<> try_send_peer() { + logger().info("[Test] try_send_peer()"); + ceph_assert(tracked_conn); + return send_op(false); + } + + seastar::future<> markdown() { + logger().info("[Test] markdown()"); + ceph_assert(tracked_conn); + tracked_conn->mark_down(); + return seastar::now(); + } + + seastar::future<> wait_blocked() { + logger().info("[Test] wait_blocked() ..."); + return interceptor.blocker.wait_blocked(); + } + + void unblock() { + logger().info("[Test] unblock()"); + return interceptor.blocker.unblock(); + } + + seastar::future<> wait_replaced(unsigned count) { + logger().info("[Test] wait_replaced({}) ...", count); + return wait_ready(0, count, false); + } + + seastar::future<> wait_established() { + logger().info("[Test] wait_established() ..."); + return wait_ready(0, 0, true); + } + + seastar::future<std::reference_wrapper<ConnResults>> + wait_results(unsigned count) { + logger().info("[Test] wait_result({}) ...", count); + return wait_ready(count, 0, true).then([this] { + return std::reference_wrapper<ConnResults>(interceptor.results); + }); + } + + bool is_standby() { + ceph_assert(tracked_conn); + return !(tracked_conn->is_connected() || tracked_conn->is_closed()); + } +}; + +class FailoverTest : public Dispatcher { + crimson::auth::DummyAuthClientServer dummy_auth; + MessengerRef cmd_msgr; + ConnectionRef cmd_conn; + const entity_addr_t test_addr; + const entity_addr_t test_peer_addr; + + std::optional<seastar::promise<>> recv_pong; + std::optional<seastar::promise<>> recv_cmdreply; + + std::unique_ptr<FailoverSuite> test_suite; + + std::optional<seastar::future<>> ms_dispatch(ConnectionRef c, MessageRef m) override { + switch (m->get_type()) { + case CEPH_MSG_PING: + ceph_assert(recv_pong); + recv_pong->set_value(); + recv_pong = std::nullopt; + break; + case MSG_COMMAND_REPLY: + ceph_assert(recv_cmdreply); + recv_cmdreply->set_value(); + recv_cmdreply = std::nullopt; + break; + case MSG_COMMAND: { + auto m_cmd = boost::static_pointer_cast<MCommand>(m); + ceph_assert(static_cast<cmd_t>(m_cmd->cmd[0][0]) == cmd_t::suite_recv_op); + ceph_assert(test_suite); + test_suite->notify_peer_reply(); + break; + } + default: + logger().error("{} got unexpected msg from cmd server: {}", *c, *m); + ceph_abort(); + } + return {seastar::now()}; + } + + private: + seastar::future<> prepare_cmd( + cmd_t cmd, + std::function<void(ceph::ref_t<MCommand>)> + f_prepare = [] (auto m) { return; }) { + assert(!recv_cmdreply); + recv_cmdreply = seastar::promise<>(); + auto fut = recv_cmdreply->get_future(); + auto m = make_message<MCommand>(); + m->cmd.emplace_back(1, static_cast<char>(cmd)); + f_prepare(m); + return cmd_conn->send(m).then([fut = std::move(fut)] () mutable { + return std::move(fut); + }); + } + + seastar::future<> start_peer(policy_t peer_policy) { + return prepare_cmd(cmd_t::suite_start, + [peer_policy] (auto m) { + m->cmd.emplace_back(1, static_cast<char>(peer_policy)); + }); + } + + seastar::future<> stop_peer() { + return prepare_cmd(cmd_t::suite_stop); + } + + seastar::future<> pingpong() { + assert(!recv_pong); + recv_pong = seastar::promise<>(); + auto fut = recv_pong->get_future(); + return cmd_conn->send(make_message<MPing>() + ).then([fut = std::move(fut)] () mutable { + return std::move(fut); + }); + } + + seastar::future<> init(entity_addr_t cmd_peer_addr) { + cmd_msgr->set_default_policy(SocketPolicy::lossy_client(0)); + cmd_msgr->set_auth_client(&dummy_auth); + cmd_msgr->set_auth_server(&dummy_auth); + return cmd_msgr->start({this}).then([this, cmd_peer_addr] { + logger().info("CmdCli connect to CmdSrv({}) ...", cmd_peer_addr); + cmd_conn = cmd_msgr->connect(cmd_peer_addr, entity_name_t::TYPE_OSD); + return pingpong(); + }); + } + + public: + FailoverTest(MessengerRef cmd_msgr, + entity_addr_t test_addr, + entity_addr_t test_peer_addr) + : cmd_msgr(cmd_msgr), + test_addr(test_addr), + test_peer_addr(test_peer_addr) { } + + seastar::future<> shutdown() { + logger().info("CmdCli shutdown..."); + assert(!recv_cmdreply); + auto m = make_message<MCommand>(); + m->cmd.emplace_back(1, static_cast<char>(cmd_t::shutdown)); + return cmd_conn->send(m).then([] { + return seastar::sleep(200ms); + }).then([this] { + cmd_msgr->stop(); + return cmd_msgr->shutdown(); + }); + } + + static seastar::future<seastar::lw_shared_ptr<FailoverTest>> + create(entity_addr_t cmd_peer_addr, entity_addr_t test_addr) { + test_addr.set_nonce(2); + cmd_peer_addr.set_nonce(3); + entity_addr_t test_peer_addr = cmd_peer_addr; + test_peer_addr.set_port(cmd_peer_addr.get_port() + 1); + test_peer_addr.set_nonce(4); + auto test = seastar::make_lw_shared<FailoverTest>( + Messenger::create(entity_name_t::OSD(1), "CmdCli", 1), + test_addr, test_peer_addr); + return test->init(cmd_peer_addr).then([test] { + logger().info("CmdCli ready"); + return test; + }); + } + + // called by tests + public: + seastar::future<> run_suite( + std::string name, + const TestInterceptor& interceptor, + policy_t test_policy, + policy_t peer_policy, + std::function<seastar::future<>(FailoverSuite&)>&& f) { + logger().info("\n\n[{}]", name); + ceph_assert(!test_suite); + SocketPolicy test_policy_ = to_socket_policy(test_policy); + return FailoverSuite::create( + test_addr, test_policy_, test_peer_addr, interceptor + ).then([this, peer_policy, f = std::move(f)] (auto suite) mutable { + ceph_assert(suite->get_addr() == test_addr); + test_suite.swap(suite); + return start_peer(peer_policy).then([this, f = std::move(f)] { + return f(*test_suite); + }).then([this] { + test_suite->post_check(); + logger().info("\n[SUCCESS]"); + }).handle_exception([this] (auto eptr) { + logger().info("\n[FAIL: {}]", eptr); + test_suite->dump_results(); + throw; + }).then([this] { + return stop_peer(); + }).then([this] { + return test_suite->shutdown().then([this] { + test_suite.reset(); + }); + }); + }); + } + + seastar::future<> peer_connect_me() { + logger().info("[Test] peer_connect_me({})", test_addr); + return prepare_cmd(cmd_t::suite_connect_me, + [this] (auto m) { + m->cmd.emplace_back(fmt::format("{}", test_addr)); + }); + } + + seastar::future<> peer_send_me() { + logger().info("[Test] peer_send_me()"); + ceph_assert(test_suite); + test_suite->needs_receive(); + return prepare_cmd(cmd_t::suite_send_me); + } + + seastar::future<> try_peer_send_me() { + logger().info("[Test] try_peer_send_me()"); + ceph_assert(test_suite); + return prepare_cmd(cmd_t::suite_send_me); + } + + seastar::future<> send_bidirectional() { + ceph_assert(test_suite); + return test_suite->send_peer().then([this] { + return peer_send_me(); + }); + } + + seastar::future<> peer_keepalive_me() { + logger().info("[Test] peer_keepalive_me()"); + ceph_assert(test_suite); + return prepare_cmd(cmd_t::suite_keepalive_me); + } + + seastar::future<> markdown_peer() { + logger().info("[Test] markdown_peer()"); + return prepare_cmd(cmd_t::suite_markdown).then([] { + // sleep awhile for peer markdown propagated + return seastar::sleep(100ms); + }); + } +}; + +class FailoverSuitePeer : public Dispatcher { + using cb_t = std::function<seastar::future<>()>; + crimson::auth::DummyAuthClientServer dummy_auth; + MessengerRef peer_msgr; + cb_t op_callback; + + ConnectionRef tracked_conn; + unsigned pending_send = 0; + + std::optional<seastar::future<>> ms_dispatch(ConnectionRef c, MessageRef m) override { + logger().info("[TestPeer] got op from Test"); + ceph_assert(m->get_type() == CEPH_MSG_OSD_OP); + ceph_assert(tracked_conn == c); + std::ignore = op_callback(); + return {seastar::now()}; + } + + void ms_handle_accept(ConnectionRef conn) override { + logger().info("[TestPeer] got accept from Test"); + ceph_assert(!tracked_conn || + tracked_conn->is_closed() || + tracked_conn == conn); + tracked_conn = conn; + std::ignore = flush_pending_send(); + } + + void ms_handle_reset(ConnectionRef conn, bool is_replace) override { + logger().info("[TestPeer] got reset from Test"); + ceph_assert(tracked_conn == conn); + tracked_conn = nullptr; + } + + private: + seastar::future<> init(entity_addr_t addr, SocketPolicy policy) { + peer_msgr->set_default_policy(policy); + peer_msgr->set_auth_client(&dummy_auth); + peer_msgr->set_auth_server(&dummy_auth); + return peer_msgr->bind(entity_addrvec_t{addr}).safe_then([this] { + return peer_msgr->start({this}); + }, Messenger::bind_ertr::all_same_way([addr] (const std::error_code& e) { + logger().error("FailoverSuitePeer: " + "there is another instance running at {}", addr); + ceph_abort(); + })); + } + + seastar::future<> send_op() { + ceph_assert(tracked_conn); + pg_t pgid; + object_locator_t oloc; + hobject_t hobj(object_t(), oloc.key, CEPH_NOSNAP, pgid.ps(), + pgid.pool(), oloc.nspace); + spg_t spgid(pgid); + return tracked_conn->send(make_message<MOSDOp>(0, 0, hobj, spgid, 0, 0, 0)); + } + + seastar::future<> flush_pending_send() { + if (pending_send != 0) { + logger().info("[TestPeer] flush sending {} ops", pending_send); + } + ceph_assert(tracked_conn); + return seastar::do_until( + [this] { return pending_send == 0; }, + [this] { + --pending_send; + return send_op(); + }); + } + + public: + FailoverSuitePeer(MessengerRef peer_msgr, cb_t op_callback) + : peer_msgr(peer_msgr), op_callback(op_callback) { } + + seastar::future<> shutdown() { + peer_msgr->stop(); + return peer_msgr->shutdown(); + } + + seastar::future<> connect_peer(entity_addr_t addr) { + logger().info("[TestPeer] connect_peer({})", addr); + auto new_tracked_conn = peer_msgr->connect(addr, entity_name_t::TYPE_OSD); + if (tracked_conn) { + if (tracked_conn->is_closed()) { + ceph_assert(tracked_conn != new_tracked_conn); + logger().info("[TestPeer] this is a new session" + " replacing an closed one"); + } else { + ceph_assert(tracked_conn == new_tracked_conn); + logger().info("[TestPeer] this is not a new session"); + } + } else { + logger().info("[TestPeer] this is a new session"); + } + tracked_conn = new_tracked_conn; + return flush_pending_send(); + } + + seastar::future<> send_peer() { + if (tracked_conn) { + logger().info("[TestPeer] send_peer()"); + return send_op(); + } else { + ++pending_send; + logger().info("[TestPeer] send_peer() (pending {})", pending_send); + return seastar::now(); + } + } + + seastar::future<> keepalive_peer() { + logger().info("[TestPeer] keepalive_peer()"); + ceph_assert(tracked_conn); + return tracked_conn->keepalive(); + } + + seastar::future<> markdown() { + logger().info("[TestPeer] markdown()"); + ceph_assert(tracked_conn); + tracked_conn->mark_down(); + return seastar::now(); + } + + static seastar::future<std::unique_ptr<FailoverSuitePeer>> + create(entity_addr_t addr, const SocketPolicy& policy, cb_t op_callback) { + auto suite = std::make_unique<FailoverSuitePeer>( + Messenger::create(entity_name_t::OSD(4), "TestPeer", 4), op_callback); + return suite->init(addr, policy + ).then([suite = std::move(suite)] () mutable { + return std::move(suite); + }); + } +}; + +class FailoverTestPeer : public Dispatcher { + crimson::auth::DummyAuthClientServer dummy_auth; + MessengerRef cmd_msgr; + ConnectionRef cmd_conn; + const entity_addr_t test_peer_addr; + std::unique_ptr<FailoverSuitePeer> test_suite; + + std::optional<seastar::future<>> ms_dispatch(ConnectionRef c, MessageRef m) override { + ceph_assert(cmd_conn == c); + switch (m->get_type()) { + case CEPH_MSG_PING: + std::ignore = c->send(make_message<MPing>()); + break; + case MSG_COMMAND: { + auto m_cmd = boost::static_pointer_cast<MCommand>(m); + auto cmd = static_cast<cmd_t>(m_cmd->cmd[0][0]); + if (cmd == cmd_t::shutdown) { + logger().info("CmdSrv shutdown..."); + // forwarded to FailoverTestPeer::wait() + cmd_msgr->stop(); + std::ignore = cmd_msgr->shutdown(); + } else { + std::ignore = handle_cmd(cmd, m_cmd).then([c] { + return c->send(make_message<MCommandReply>()); + }); + } + break; + } + default: + logger().error("{} got unexpected msg from cmd client: {}", *c, m); + ceph_abort(); + } + return {seastar::now()}; + } + + void ms_handle_accept(ConnectionRef conn) override { + cmd_conn = conn; + } + + private: + seastar::future<> notify_recv_op() { + ceph_assert(cmd_conn); + auto m = make_message<MCommand>(); + m->cmd.emplace_back(1, static_cast<char>(cmd_t::suite_recv_op)); + return cmd_conn->send(m); + } + + seastar::future<> handle_cmd(cmd_t cmd, MRef<MCommand> m_cmd) { + switch (cmd) { + case cmd_t::suite_start: { + ceph_assert(!test_suite); + auto policy = to_socket_policy(static_cast<policy_t>(m_cmd->cmd[1][0])); + return FailoverSuitePeer::create(test_peer_addr, policy, + [this] { return notify_recv_op(); } + ).then([this] (auto suite) { + test_suite.swap(suite); + }); + } + case cmd_t::suite_stop: + ceph_assert(test_suite); + return test_suite->shutdown().then([this] { + test_suite.reset(); + }); + case cmd_t::suite_connect_me: { + ceph_assert(test_suite); + entity_addr_t test_addr = entity_addr_t(); + test_addr.parse(m_cmd->cmd[1].c_str(), nullptr); + return test_suite->connect_peer(test_addr); + } + case cmd_t::suite_send_me: + ceph_assert(test_suite); + return test_suite->send_peer(); + case cmd_t::suite_keepalive_me: + ceph_assert(test_suite); + return test_suite->keepalive_peer(); + case cmd_t::suite_markdown: + ceph_assert(test_suite); + return test_suite->markdown(); + default: + logger().error("TestPeer got unexpected command {} from Test", m_cmd); + ceph_abort(); + return seastar::now(); + } + } + + seastar::future<> init(entity_addr_t cmd_peer_addr) { + cmd_msgr->set_default_policy(SocketPolicy::stateless_server(0)); + cmd_msgr->set_auth_client(&dummy_auth); + cmd_msgr->set_auth_server(&dummy_auth); + return cmd_msgr->bind(entity_addrvec_t{cmd_peer_addr}).safe_then([this] { + return cmd_msgr->start({this}); + }, Messenger::bind_ertr::all_same_way([cmd_peer_addr] (const std::error_code& e) { + logger().error("FailoverTestPeer: " + "there is another instance running at {}", cmd_peer_addr); + ceph_abort(); + })); + } + + public: + FailoverTestPeer(MessengerRef cmd_msgr, + entity_addr_t test_peer_addr) + : cmd_msgr(cmd_msgr), + test_peer_addr(test_peer_addr) { } + + seastar::future<> wait() { + return cmd_msgr->wait(); + } + + static seastar::future<std::unique_ptr<FailoverTestPeer>> + create(entity_addr_t cmd_peer_addr) { + // suite bind to cmd_peer_addr, with port + 1 + entity_addr_t test_peer_addr = cmd_peer_addr; + test_peer_addr.set_port(cmd_peer_addr.get_port() + 1); + auto test_peer = std::make_unique<FailoverTestPeer>( + Messenger::create(entity_name_t::OSD(3), "CmdSrv", 3), test_peer_addr); + return test_peer->init(cmd_peer_addr + ).then([test_peer = std::move(test_peer)] () mutable { + logger().info("CmdSrv ready"); + return std::move(test_peer); + }); + } +}; + +seastar::future<> +test_v2_lossy_early_connect_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {custom_bp_t::BANNER_WRITE}, + {custom_bp_t::BANNER_READ}, + {custom_bp_t::BANNER_PAYLOAD_READ}, + {custom_bp_t::SOCKET_CONNECTING}, + {Tag::HELLO, bp_type_t::WRITE}, + {Tag::HELLO, bp_type_t::READ}, + {Tag::AUTH_REQUEST, bp_type_t::WRITE}, + {Tag::AUTH_DONE, bp_type_t::READ}, + {Tag::AUTH_SIGNATURE, bp_type_t::WRITE}, + {Tag::AUTH_SIGNATURE, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossy_early_connect_fault -- {}", bp), + interceptor, + policy_t::lossy_client, + policy_t::stateless_server, + [] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossy_connect_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::CLIENT_IDENT, bp_type_t::WRITE}, + {Tag::SERVER_IDENT, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossy_connect_fault -- {}", bp), + interceptor, + policy_t::lossy_client, + policy_t::stateless_server, + [] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 2, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossy_connected_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::MESSAGE, bp_type_t::WRITE}, + {Tag::MESSAGE, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossy_connected_fault -- {}", bp), + interceptor, + policy_t::lossy_client, + policy_t::stateless_server, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(1, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossy_early_accept_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {custom_bp_t::BANNER_WRITE}, + {custom_bp_t::BANNER_READ}, + {custom_bp_t::BANNER_PAYLOAD_READ}, + {Tag::HELLO, bp_type_t::WRITE}, + {Tag::HELLO, bp_type_t::READ}, + {Tag::AUTH_REQUEST, bp_type_t::READ}, + {Tag::AUTH_DONE, bp_type_t::WRITE}, + {Tag::AUTH_SIGNATURE, bp_type_t::WRITE}, + {Tag::AUTH_SIGNATURE, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossy_early_accept_fault -- {}", bp), + interceptor, + policy_t::stateless_server, + policy_t::lossy_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossy_accept_fault(FailoverTest& test) { + auto bp = Breakpoint{Tag::CLIENT_IDENT, bp_type_t::READ}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossy_accept_fault -- {}", bp), + interceptor, + policy_t::stateless_server, + policy_t::lossy_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_lossy_establishing_fault(FailoverTest& test) { + auto bp = Breakpoint{Tag::SERVER_IDENT, bp_type_t::WRITE}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossy_establishing_fault -- {}", bp), + interceptor, + policy_t::stateless_server, + policy_t::lossy_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(1, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_lossy_accepted_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::MESSAGE, bp_type_t::WRITE}, + {Tag::MESSAGE, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossy_accepted_fault -- {}", bp), + interceptor, + policy_t::stateless_server, + policy_t::lossy_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(1, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossless_connect_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::CLIENT_IDENT, bp_type_t::WRITE}, + {Tag::SERVER_IDENT, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossless_connect_fault -- {}", bp), + interceptor, + policy_t::lossless_client, + policy_t::stateful_server, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 2, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossless_connected_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::MESSAGE, bp_type_t::WRITE}, + {Tag::MESSAGE, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossless_connected_fault -- {}", bp), + interceptor, + policy_t::lossless_client, + policy_t::stateful_server, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 1, 1, 2); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossless_connected_fault2(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::ACK, bp_type_t::READ}, + {Tag::ACK, bp_type_t::WRITE}, + {Tag::KEEPALIVE2, bp_type_t::READ}, + {Tag::KEEPALIVE2, bp_type_t::WRITE}, + {Tag::KEEPALIVE2_ACK, bp_type_t::READ}, + {Tag::KEEPALIVE2_ACK, bp_type_t::WRITE}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossless_connected_fault2 -- {}", bp), + interceptor, + policy_t::lossless_client, + policy_t::stateful_server, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_established(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.keepalive_peer(); + }).then([&suite] { + return suite.wait_established(); + }).then([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_keepalive_me(); + }).then([&suite] { + return suite.wait_established(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_established(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_established(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 1, 1, 2); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossless_reconnect_fault(FailoverTest& test) { + return seastar::do_with(std::vector<std::pair<Breakpoint, Breakpoint>>{ + {{Tag::MESSAGE, bp_type_t::WRITE}, + {Tag::SESSION_RECONNECT, bp_type_t::WRITE}}, + {{Tag::MESSAGE, bp_type_t::WRITE}, + {Tag::SESSION_RECONNECT_OK, bp_type_t::READ}}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp_pair) { + TestInterceptor interceptor; + interceptor.make_fault(bp_pair.first); + interceptor.make_fault(bp_pair.second); + return test.run_suite( + fmt::format("test_v2_lossless_reconnect_fault -- {}, {}", + bp_pair.first, bp_pair.second), + interceptor, + policy_t::lossless_client, + policy_t::stateful_server, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(3, 1, 2, 2); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossless_accept_fault(FailoverTest& test) { + auto bp = Breakpoint{Tag::CLIENT_IDENT, bp_type_t::READ}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossless_accept_fault -- {}", bp), + interceptor, + policy_t::stateful_server, + policy_t::lossless_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_lossless_establishing_fault(FailoverTest& test) { + auto bp = Breakpoint{Tag::SERVER_IDENT, bp_type_t::WRITE}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossless_establishing_fault -- {}", bp), + interceptor, + policy_t::stateful_server, + policy_t::lossless_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_lossless_accepted_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::MESSAGE, bp_type_t::WRITE}, + {Tag::MESSAGE, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_lossless_accepted_fault -- {}", bp), + interceptor, + policy_t::stateful_server, + policy_t::lossless_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0); + results[1].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_lossless_reaccept_fault(FailoverTest& test) { + return seastar::do_with(std::vector<std::pair<Breakpoint, Breakpoint>>{ + {{Tag::MESSAGE, bp_type_t::READ}, + {Tag::SESSION_RECONNECT, bp_type_t::READ}}, + {{Tag::MESSAGE, bp_type_t::READ}, + {Tag::SESSION_RECONNECT_OK, bp_type_t::WRITE}}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp_pair) { + TestInterceptor interceptor; + interceptor.make_fault(bp_pair.first); + interceptor.make_fault(bp_pair.second); + return test.run_suite( + fmt::format("test_v2_lossless_reaccept_fault -- {}, {}", + bp_pair.first, bp_pair.second), + interceptor, + policy_t::stateful_server, + policy_t::lossless_client, + [&test, bp = bp_pair.second] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.send_bidirectional(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(3); + }).then([bp] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + if (bp == Breakpoint{Tag::SESSION_RECONNECT, bp_type_t::READ}) { + results[0].assert_accept(1, 1, 0, 2); + } else { + results[0].assert_accept(1, 1, 0, 3); + } + results[0].assert_reset(0, 0); + if (bp == Breakpoint{Tag::SESSION_RECONNECT, bp_type_t::READ}) { + results[1].assert_state_at(conn_state_t::closed); + } else { + results[1].assert_state_at(conn_state_t::replaced); + } + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0, 1, 0); + results[1].assert_reset(0, 0); + results[2].assert_state_at(conn_state_t::replaced); + results[2].assert_connect(0, 0, 0, 0); + results[2].assert_accept(1, 0, 1, 0); + results[2].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_peer_connect_fault(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {Tag::CLIENT_IDENT, bp_type_t::WRITE}, + {Tag::SERVER_IDENT, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_peer_connect_fault -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 2, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_peer_accept_fault(FailoverTest& test) { + auto bp = Breakpoint{Tag::CLIENT_IDENT, bp_type_t::READ}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_peer_accept_fault -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_peer_establishing_fault(FailoverTest& test) { + auto bp = Breakpoint{Tag::SERVER_IDENT, bp_type_t::WRITE}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_peer_establishing_fault -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_peer_connected_fault_reconnect(FailoverTest& test) { + auto bp = Breakpoint{Tag::MESSAGE, bp_type_t::WRITE}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_peer_connected_fault_reconnect -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 1, 1, 2); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_peer_connected_fault_reaccept(FailoverTest& test) { + auto bp = Breakpoint{Tag::MESSAGE, bp_type_t::READ}; + TestInterceptor interceptor; + interceptor.make_fault(bp); + return test.run_suite( + fmt::format("test_v2_peer_connected_fault_reaccept -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 1); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0, 1, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<bool> +peer_wins(FailoverTest& test) { + return seastar::do_with(bool(), [&test] (auto& ret) { + return test.run_suite("peer_wins", + TestInterceptor(), + policy_t::lossy_client, + policy_t::stateless_server, + [&ret] (FailoverSuite& suite) { + return suite.connect_peer().then([&suite] { + return suite.wait_results(1); + }).then([&ret] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + ret = results[0].conn->peer_wins(); + logger().info("peer_wins: {}", ret); + }); + }).then([&ret] { + return ret; + }); + }); +} + +seastar::future<> +test_v2_racing_reconnect_win(FailoverTest& test) { + return seastar::do_with(std::vector<std::pair<unsigned, Breakpoint>>{ + {2, {custom_bp_t::BANNER_WRITE}}, + {2, {custom_bp_t::BANNER_READ}}, + {2, {custom_bp_t::BANNER_PAYLOAD_READ}}, + {2, {Tag::HELLO, bp_type_t::WRITE}}, + {2, {Tag::HELLO, bp_type_t::READ}}, + {2, {Tag::AUTH_REQUEST, bp_type_t::READ}}, + {2, {Tag::AUTH_DONE, bp_type_t::WRITE}}, + {2, {Tag::AUTH_SIGNATURE, bp_type_t::WRITE}}, + {2, {Tag::AUTH_SIGNATURE, bp_type_t::READ}}, + {1, {Tag::SESSION_RECONNECT, bp_type_t::READ}}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault({Tag::MESSAGE, bp_type_t::READ}); + interceptor.make_block(bp.second, bp.first); + return test.run_suite( + fmt::format("test_v2_racing_reconnect_win -- {}({})", + bp.second, bp.first), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_established(); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 0, 1, 1); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::closed); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0); + results[1].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_racing_reconnect_lose(FailoverTest& test) { + return seastar::do_with(std::vector<std::pair<unsigned, Breakpoint>>{ + {2, {custom_bp_t::BANNER_WRITE}}, + {2, {custom_bp_t::BANNER_READ}}, + {2, {custom_bp_t::BANNER_PAYLOAD_READ}}, + {2, {Tag::HELLO, bp_type_t::WRITE}}, + {2, {Tag::HELLO, bp_type_t::READ}}, + {2, {Tag::AUTH_REQUEST, bp_type_t::WRITE}}, + {2, {Tag::AUTH_DONE, bp_type_t::READ}}, + {2, {Tag::AUTH_SIGNATURE, bp_type_t::WRITE}}, + {2, {Tag::AUTH_SIGNATURE, bp_type_t::READ}}, + {1, {Tag::SESSION_RECONNECT, bp_type_t::WRITE}}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_fault({Tag::MESSAGE, bp_type_t::WRITE}); + interceptor.make_block(bp.second, bp.first); + return test.run_suite( + fmt::format("test_v2_racing_reconnect_lose -- {}({})", + bp.second, bp.first), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_replaced(1); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 1); + results[0].assert_accept(0, 0, 0, 1); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0, 1, 0); + results[1].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_racing_connect_win(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {custom_bp_t::BANNER_WRITE}, + {custom_bp_t::BANNER_READ}, + {custom_bp_t::BANNER_PAYLOAD_READ}, + {Tag::HELLO, bp_type_t::WRITE}, + {Tag::HELLO, bp_type_t::READ}, + {Tag::AUTH_REQUEST, bp_type_t::READ}, + {Tag::AUTH_DONE, bp_type_t::WRITE}, + {Tag::AUTH_SIGNATURE, bp_type_t::WRITE}, + {Tag::AUTH_SIGNATURE, bp_type_t::READ}, + {Tag::CLIENT_IDENT, bp_type_t::READ}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_block(bp); + return test.run_suite( + fmt::format("test_v2_racing_connect_win -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_established(); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(1, 1, 0, 1); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_racing_connect_lose(FailoverTest& test) { + return seastar::do_with(std::vector<Breakpoint>{ + {custom_bp_t::BANNER_WRITE}, + {custom_bp_t::BANNER_READ}, + {custom_bp_t::BANNER_PAYLOAD_READ}, + {Tag::HELLO, bp_type_t::WRITE}, + {Tag::HELLO, bp_type_t::READ}, + {Tag::AUTH_REQUEST, bp_type_t::WRITE}, + {Tag::AUTH_DONE, bp_type_t::READ}, + {Tag::AUTH_SIGNATURE, bp_type_t::WRITE}, + {Tag::AUTH_SIGNATURE, bp_type_t::READ}, + {Tag::CLIENT_IDENT, bp_type_t::WRITE}, + }, [&test] (auto& failure_cases) { + return seastar::do_for_each(failure_cases, [&test] (auto bp) { + TestInterceptor interceptor; + interceptor.make_block(bp); + return test.run_suite( + fmt::format("test_v2_racing_connect_lose -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_replaced(1); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 0); + results[0].assert_accept(0, 0, 0, 1); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }); + }); + }); + }); +} + +seastar::future<> +test_v2_racing_connect_reconnect_lose(FailoverTest& test) { + TestInterceptor interceptor; + interceptor.make_fault({Tag::SERVER_IDENT, bp_type_t::READ}); + interceptor.make_block({Tag::CLIENT_IDENT, bp_type_t::WRITE}, 2); + return test.run_suite("test_v2_racing_connect_reconnect_lose", + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_replaced(1); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 2, 0, 0); + results[0].assert_accept(0, 0, 0, 1); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 1, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_racing_connect_reconnect_win(FailoverTest& test) { + TestInterceptor interceptor; + interceptor.make_fault({Tag::SERVER_IDENT, bp_type_t::READ}); + interceptor.make_block({Tag::SESSION_RECONNECT, bp_type_t::READ}); + return test.run_suite("test_v2_racing_connect_reconnect_win", + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_established(); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 2, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::closed); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0, 1, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_stale_connect(FailoverTest& test) { + auto bp = Breakpoint{Tag::SERVER_IDENT, bp_type_t::READ}; + TestInterceptor interceptor; + interceptor.make_stall(bp); + return test.run_suite( + fmt::format("test_v2_stale_connect -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_replaced(1); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 1, 0, 0); + results[0].assert_accept(0, 0, 0, 1); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 1, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_stale_reconnect(FailoverTest& test) { + auto bp = Breakpoint{Tag::SESSION_RECONNECT_OK, bp_type_t::READ}; + TestInterceptor interceptor; + interceptor.make_fault({Tag::MESSAGE, bp_type_t::WRITE}); + interceptor.make_stall(bp); + return test.run_suite( + fmt::format("test_v2_stale_reconnect -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_replaced(1); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(2, 1, 1, 1); + results[0].assert_accept(0, 0, 0, 1); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0, 1, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_stale_accept(FailoverTest& test) { + auto bp = Breakpoint{Tag::CLIENT_IDENT, bp_type_t::READ}; + TestInterceptor interceptor; + interceptor.make_stall(bp); + return test.run_suite( + fmt::format("test_v2_stale_accept -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_established(); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_stale_establishing(FailoverTest& test) { + auto bp = Breakpoint{Tag::SERVER_IDENT, bp_type_t::WRITE}; + TestInterceptor interceptor; + interceptor.make_stall(bp); + return test.run_suite( + fmt::format("test_v2_stale_establishing -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_replaced(1); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0); + results[1].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_stale_reaccept(FailoverTest& test) { + auto bp = Breakpoint{Tag::SESSION_RECONNECT_OK, bp_type_t::WRITE}; + TestInterceptor interceptor; + interceptor.make_fault({Tag::MESSAGE, bp_type_t::READ}); + interceptor.make_stall(bp); + return test.run_suite( + fmt::format("test_v2_stale_reaccept -- {}", bp), + interceptor, + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + return test.peer_send_me(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&suite] { + return suite.wait_blocked(); + }).then([] { + logger().info("[Test] block the broken REPLACING for 210ms..."); + return seastar::sleep(210ms); + }).then([&suite] { + suite.unblock(); + return suite.wait_results(3); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 3); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 0, 1, 0); + results[1].assert_reset(0, 0); + results[2].assert_state_at(conn_state_t::replaced); + results[2].assert_connect(0, 0, 0, 0); + results[2].assert_accept(1, 0); + results[2].assert_reset(0, 0); + ceph_assert(results[2].server_reconnect_attempts >= 1); + }); + }); +} + +seastar::future<> +test_v2_lossy_client(FailoverTest& test) { + return test.run_suite( + "test_v2_lossy_client", + TestInterceptor(), + policy_t::lossy_client, + policy_t::stateless_server, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return suite.connect_peer(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }).then([&suite] { + logger().info("-- 1 --"); + logger().info("[Test] client markdown..."); + return suite.markdown(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(1, 1, 0, 1); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 2 --"); + logger().info("[Test] server markdown..."); + return test.markdown_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::closed); + results[1].assert_connect(1, 1, 0, 1); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(1, 0); + }).then([&suite] { + logger().info("-- 3 --"); + logger().info("[Test] client reconnect..."); + return suite.connect_peer(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_results(3); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::closed); + results[1].assert_connect(1, 1, 0, 1); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(1, 0); + results[2].assert_state_at(conn_state_t::established); + results[2].assert_connect(1, 1, 0, 1); + results[2].assert_accept(0, 0, 0, 0); + results[2].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_stateless_server(FailoverTest& test) { + return test.run_suite( + "test_v2_stateless_server", + TestInterceptor(), + policy_t::stateless_server, + policy_t::lossy_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return test.peer_connect_me(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 1 --"); + logger().info("[Test] client markdown..."); + return test.markdown_peer(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(1, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }).then([&suite] { + logger().info("-- 2 --"); + logger().info("[Test] server markdown..."); + return suite.markdown(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(1, 0); + results[1].assert_state_at(conn_state_t::closed); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 3 --"); + logger().info("[Test] client reconnect..."); + return test.peer_connect_me(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_results(3); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(1, 0); + results[1].assert_state_at(conn_state_t::closed); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 1); + results[1].assert_reset(0, 0); + results[2].assert_state_at(conn_state_t::established); + results[2].assert_connect(0, 0, 0, 0); + results[2].assert_accept(1, 1, 0, 1); + results[2].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_lossless_client(FailoverTest& test) { + return test.run_suite( + "test_v2_lossless_client", + TestInterceptor(), + policy_t::lossless_client, + policy_t::stateful_server, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return suite.connect_peer(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }).then([&suite] { + logger().info("-- 1 --"); + logger().info("[Test] client markdown..."); + return suite.markdown(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(1, 1, 0, 1); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 2 --"); + logger().info("[Test] server markdown..."); + return test.markdown_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(2, 2, 1, 2); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 1); + }).then([&suite] { + logger().info("-- 3 --"); + logger().info("[Test] client reconnect..."); + return suite.connect_peer(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(2, 2, 1, 2); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 1); + }); + }); +} + +seastar::future<> +test_v2_stateful_server(FailoverTest& test) { + return test.run_suite( + "test_v2_stateful_server", + TestInterceptor(), + policy_t::stateful_server, + policy_t::lossless_client, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return test.peer_connect_me(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 1 --"); + logger().info("[Test] client markdown..."); + return test.markdown_peer(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 1); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }).then([&suite] { + logger().info("-- 2 --"); + logger().info("[Test] server markdown..."); + return suite.markdown(); + }).then([&suite] { + return suite.wait_results(3); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 1); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + results[2].assert_state_at(conn_state_t::established); + results[2].assert_connect(0, 0, 0, 0); + results[2].assert_accept(1, 1, 1, 1); + results[2].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 3 --"); + logger().info("[Test] client reconnect..."); + return test.peer_connect_me(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_results(3); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 1); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + results[2].assert_state_at(conn_state_t::established); + results[2].assert_connect(0, 0, 0, 0); + results[2].assert_accept(1, 1, 1, 1); + results[2].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_peer_reuse_connector(FailoverTest& test) { + return test.run_suite( + "test_v2_peer_reuse_connector", + TestInterceptor(), + policy_t::lossless_peer_reuse, + policy_t::lossless_peer_reuse, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return suite.connect_peer(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }).then([&suite] { + logger().info("-- 1 --"); + logger().info("[Test] connector markdown..."); + return suite.markdown(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(1, 1, 0, 1); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 2 --"); + logger().info("[Test] acceptor markdown..."); + return test.markdown_peer(); + }).then([] { + return seastar::sleep(100ms); + }).then([&suite] { + ceph_assert(suite.is_standby()); + logger().info("-- 3 --"); + logger().info("[Test] connector reconnect..."); + return suite.connect_peer(); + }).then([&suite] { + return suite.try_send_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(2, 2, 1, 2); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 1); + }); + }); +} + +seastar::future<> +test_v2_peer_reuse_acceptor(FailoverTest& test) { + return test.run_suite( + "test_v2_peer_reuse_acceptor", + TestInterceptor(), + policy_t::lossless_peer_reuse, + policy_t::lossless_peer_reuse, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return test.peer_connect_me(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 1 --"); + logger().info("[Test] connector markdown..."); + return test.markdown_peer(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 1); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }).then([] { + logger().info("-- 2 --"); + logger().info("[Test] acceptor markdown..."); + return seastar::sleep(100ms); + }).then([&suite] { + return suite.markdown(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 1); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 3 --"); + logger().info("[Test] connector reconnect..."); + return test.peer_connect_me(); + }).then([&test] { + return test.try_peer_send_me(); + }).then([&suite] { + return suite.wait_results(3); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 1); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + results[2].assert_state_at(conn_state_t::established); + results[2].assert_connect(0, 0, 0, 0); + results[2].assert_accept(1, 1, 1, 1); + results[2].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_lossless_peer_connector(FailoverTest& test) { + return test.run_suite( + "test_v2_lossless_peer_connector", + TestInterceptor(), + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&suite] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return suite.connect_peer(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + }).then([&suite] { + logger().info("-- 1 --"); + logger().info("[Test] connector markdown..."); + return suite.markdown(); + }).then([&suite] { + return suite.connect_peer(); + }).then([&suite] { + return suite.send_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(1, 1, 0, 1); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 2 --"); + logger().info("[Test] acceptor markdown..."); + return test.markdown_peer(); + }).then([] { + return seastar::sleep(100ms); + }).then([&suite] { + ceph_assert(suite.is_standby()); + logger().info("-- 3 --"); + logger().info("[Test] connector reconnect..."); + return suite.connect_peer(); + }).then([&suite] { + return suite.try_send_peer(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(1, 1, 0, 1); + results[0].assert_accept(0, 0, 0, 0); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::established); + results[1].assert_connect(2, 2, 1, 2); + results[1].assert_accept(0, 0, 0, 0); + results[1].assert_reset(0, 1); + }); + }); +} + +seastar::future<> +test_v2_lossless_peer_acceptor(FailoverTest& test) { + return test.run_suite( + "test_v2_lossless_peer_acceptor", + TestInterceptor(), + policy_t::lossless_peer, + policy_t::lossless_peer, + [&test] (FailoverSuite& suite) { + return seastar::futurize_invoke([&test] { + logger().info("-- 0 --"); + logger().info("[Test] setup connection..."); + return test.peer_connect_me(); + }).then([&test] { + return test.send_bidirectional(); + }).then([&suite] { + return suite.wait_results(1); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 1); + results[0].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 1 --"); + logger().info("[Test] connector markdown..."); + return test.markdown_peer(); + }).then([&test] { + return test.peer_connect_me(); + }).then([&test] { + return test.peer_send_me(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::established); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }).then([] { + logger().info("-- 2 --"); + logger().info("[Test] acceptor markdown..."); + return seastar::sleep(100ms); + }).then([&suite] { + return suite.markdown(); + }).then([&suite] { + return suite.wait_results(2); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + }).then([&test] { + logger().info("-- 3 --"); + logger().info("[Test] connector reconnect..."); + return test.peer_connect_me(); + }).then([&test] { + return test.try_peer_send_me(); + }).then([&suite] { + return suite.wait_results(3); + }).then([] (ConnResults& results) { + results[0].assert_state_at(conn_state_t::closed); + results[0].assert_connect(0, 0, 0, 0); + results[0].assert_accept(1, 1, 0, 2); + results[0].assert_reset(0, 0); + results[1].assert_state_at(conn_state_t::replaced); + results[1].assert_connect(0, 0, 0, 0); + results[1].assert_accept(1, 1, 0, 0); + results[1].assert_reset(0, 0); + results[2].assert_state_at(conn_state_t::established); + results[2].assert_connect(0, 0, 0, 0); + results[2].assert_accept(1, 1, 1, 1); + results[2].assert_reset(0, 0); + }); + }); +} + +seastar::future<> +test_v2_protocol(entity_addr_t test_addr, + entity_addr_t test_peer_addr, + bool test_peer_islocal) { + ceph_assert(test_addr.is_msgr2()); + ceph_assert(test_peer_addr.is_msgr2()); + + if (test_peer_islocal) { + // initiate crimson test peer locally + logger().info("test_v2_protocol: start local TestPeer at {}...", test_peer_addr); + return FailoverTestPeer::create(test_peer_addr + ).then([test_addr, test_peer_addr] (auto peer) { + return test_v2_protocol(test_addr, test_peer_addr, false + ).then([peer = std::move(peer)] () mutable { + return peer->wait().then([peer = std::move(peer)] {}); + }); + }).handle_exception([] (auto eptr) { + logger().error("FailoverTestPeer failed: got exception {}", eptr); + throw; + }); + } + + return FailoverTest::create(test_peer_addr, test_addr).then([] (auto test) { + return seastar::futurize_invoke([test] { + return test_v2_lossy_early_connect_fault(*test); + }).then([test] { + return test_v2_lossy_connect_fault(*test); + }).then([test] { + return test_v2_lossy_connected_fault(*test); + }).then([test] { + return test_v2_lossy_early_accept_fault(*test); + }).then([test] { + return test_v2_lossy_accept_fault(*test); + }).then([test] { + return test_v2_lossy_establishing_fault(*test); + }).then([test] { + return test_v2_lossy_accepted_fault(*test); + }).then([test] { + return test_v2_lossless_connect_fault(*test); + }).then([test] { + return test_v2_lossless_connected_fault(*test); + }).then([test] { + return test_v2_lossless_connected_fault2(*test); + }).then([test] { + return test_v2_lossless_reconnect_fault(*test); + }).then([test] { + return test_v2_lossless_accept_fault(*test); + }).then([test] { + return test_v2_lossless_establishing_fault(*test); + }).then([test] { + return test_v2_lossless_accepted_fault(*test); + }).then([test] { + return test_v2_lossless_reaccept_fault(*test); + }).then([test] { + return test_v2_peer_connect_fault(*test); + }).then([test] { + return test_v2_peer_accept_fault(*test); + }).then([test] { + return test_v2_peer_establishing_fault(*test); + }).then([test] { + return test_v2_peer_connected_fault_reconnect(*test); + }).then([test] { + return test_v2_peer_connected_fault_reaccept(*test); + }).then([test] { + return peer_wins(*test); + }).then([test] (bool peer_wins) { + if (peer_wins) { + return seastar::futurize_invoke([test] { + return test_v2_racing_connect_lose(*test); + }).then([test] { + return test_v2_racing_reconnect_lose(*test); + }); + } else { + return seastar::futurize_invoke([test] { + return test_v2_racing_connect_win(*test); + }).then([test] { + return test_v2_racing_reconnect_win(*test); + }); + } + }).then([test] { + return test_v2_racing_connect_reconnect_win(*test); + }).then([test] { + return test_v2_racing_connect_reconnect_lose(*test); + }).then([test] { + return test_v2_stale_connect(*test); + }).then([test] { + return test_v2_stale_reconnect(*test); + }).then([test] { + return test_v2_stale_accept(*test); + }).then([test] { + return test_v2_stale_establishing(*test); + }).then([test] { + return test_v2_stale_reaccept(*test); + }).then([test] { + return test_v2_lossy_client(*test); + }).then([test] { + return test_v2_stateless_server(*test); + }).then([test] { + return test_v2_lossless_client(*test); + }).then([test] { + return test_v2_stateful_server(*test); + }).then([test] { + return test_v2_peer_reuse_connector(*test); + }).then([test] { + return test_v2_peer_reuse_acceptor(*test); + }).then([test] { + return test_v2_lossless_peer_connector(*test); + }).then([test] { + return test_v2_lossless_peer_acceptor(*test); + }).then([test] { + return test->shutdown().then([test] {}); + }); + }).handle_exception([] (auto eptr) { + logger().error("FailoverTest failed: got exception {}", eptr); + throw; + }); +} + +} + +seastar::future<int> do_test(seastar::app_template& app) +{ + std::vector<const char*> args; + std::string cluster; + std::string conf_file_list; + auto init_params = ceph_argparse_early_args(args, + CEPH_ENTITY_TYPE_CLIENT, + &cluster, + &conf_file_list); + return crimson::common::sharded_conf().start(init_params.name, cluster) + .then([conf_file_list] { + return local_conf().parse_config_files(conf_file_list); + }).then([&app] { + auto&& config = app.configuration(); + verbose = config["verbose"].as<bool>(); + auto rounds = config["rounds"].as<unsigned>(); + auto keepalive_ratio = config["keepalive-ratio"].as<double>(); + entity_addr_t v2_test_addr; + ceph_assert(v2_test_addr.parse( + config["v2-test-addr"].as<std::string>().c_str(), nullptr)); + entity_addr_t v2_testpeer_addr; + ceph_assert(v2_testpeer_addr.parse( + config["v2-testpeer-addr"].as<std::string>().c_str(), nullptr)); + auto v2_testpeer_islocal = config["v2-testpeer-islocal"].as<bool>(); + return test_echo(rounds, keepalive_ratio, false) + .then([rounds, keepalive_ratio] { + return test_echo(rounds, keepalive_ratio, true); + }).then([] { + return test_concurrent_dispatch(false); + }).then([] { + return test_concurrent_dispatch(true); + }).then([] { + return test_preemptive_shutdown(false); + }).then([] { + return test_preemptive_shutdown(true); + }).then([v2_test_addr, v2_testpeer_addr, v2_testpeer_islocal] { + return test_v2_protocol(v2_test_addr, v2_testpeer_addr, v2_testpeer_islocal); + }).then([] { + logger().info("All tests succeeded"); + // Seastar has bugs to have events undispatched during shutdown, + // which will result in memory leak and thus fail LeakSanitizer. + return seastar::sleep(100ms); + }); + }).then([] { + return crimson::common::sharded_conf().stop(); + }).then([] { + return 0; + }).handle_exception([] (auto eptr) { + logger().error("Test failed: got exception {}", eptr); + return 1; + }); +} + +int main(int argc, char** argv) +{ + seastar::app_template app; + app.add_options() + ("verbose,v", bpo::value<bool>()->default_value(false), + "chatty if true") + ("rounds", bpo::value<unsigned>()->default_value(512), + "number of pingpong rounds") + ("keepalive-ratio", bpo::value<double>()->default_value(0.1), + "ratio of keepalive in ping messages") + ("v2-test-addr", bpo::value<std::string>()->default_value("v2:127.0.0.1:9012"), + "address of v2 failover tests") + ("v2-testpeer-addr", bpo::value<std::string>()->default_value("v2:127.0.0.1:9013"), + "addresses of v2 failover testpeer" + " (CmdSrv address and TestPeer address with port+=1)") + ("v2-testpeer-islocal", bpo::value<bool>()->default_value(true), + "create a local crimson testpeer, or connect to a remote testpeer"); + return app.run(argc, argv, [&app] { + // This test normally succeeds within 60 seconds, so kill it after 120 + // seconds in case it is blocked forever due to unaddressed bugs. + return seastar::with_timeout(seastar::lowres_clock::now() + 120s, do_test(app)) + .handle_exception_type([](seastar::timed_out_error&) { + logger().error("test_messenger timeout after 120s, abort! " + "Consider to extend the period if the test is still running."); + // use the retcode of timeout(1) + return 124; + }); + }); +} diff --git a/src/test/crimson/test_messenger_peer.cc b/src/test/crimson/test_messenger_peer.cc new file mode 100644 index 000000000..7b669675b --- /dev/null +++ b/src/test/crimson/test_messenger_peer.cc @@ -0,0 +1,447 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- + +#include <boost/pointer_cast.hpp> +#include <boost/program_options/variables_map.hpp> +#include <boost/program_options/parsers.hpp> + +#include "auth/DummyAuth.h" +#include "common/dout.h" +#include "global/global_init.h" +#include "messages/MPing.h" +#include "messages/MCommand.h" +#include "messages/MCommandReply.h" +#include "messages/MOSDOp.h" +#include "msg/Dispatcher.h" +#include "msg/Messenger.h" + +#include "test_cmds.h" + +namespace { + +#define dout_subsys ceph_subsys_test + +using ceph::net::test::cmd_t; +using ceph::net::test::policy_t; +using SocketPolicy = Messenger::Policy; + +constexpr int CEPH_OSD_PROTOCOL = 10; + +class FailoverSuitePeer : public Dispatcher { + using cb_t = std::function<void()>; + DummyAuthClientServer dummy_auth; + std::unique_ptr<Messenger> peer_msgr; + cb_t op_callback; + + Connection* tracked_conn = nullptr; + unsigned pending_send = 0; + + bool ms_can_fast_dispatch_any() const override { return true; } + bool ms_can_fast_dispatch(const Message* m) const override { return true; } + void ms_fast_dispatch(Message* m) override { + auto conn = m->get_connection().get(); + if (tracked_conn == nullptr) { + ldout(cct, 0) << "[!TestPeer] got op from Test(conn " + << conn << "not tracked yet)" << dendl; + tracked_conn = conn; + } else if (tracked_conn != conn) { + lderr(cct) << "[TestPeer] got op from Test: conn(" << conn + << ") != tracked_conn(" << tracked_conn + << ")" << dendl; + ceph_abort(); + } else { + ldout(cct, 0) << "[TestPeer] got op from Test" << dendl; + } + op_callback(); + } + bool ms_dispatch(Message* m) override { ceph_abort(); } + void ms_handle_fast_connect(Connection* conn) override { + if (tracked_conn == conn) { + ldout(cct, 0) << "[TestPeer] connected: " << conn << dendl; + } else { + lderr(cct) << "[TestPeer] connected: conn(" << conn + << ") != tracked_conn(" << tracked_conn + << ")" << dendl; + ceph_abort(); + } + } + void ms_handle_fast_accept(Connection* conn) override { + if (tracked_conn == nullptr) { + ldout(cct, 0) << "[TestPeer] accepted: " << conn << dendl; + tracked_conn = conn; + } else if (tracked_conn != conn) { + lderr(cct) << "[TestPeer] accepted: conn(" << conn + << ") != tracked_conn(" << tracked_conn + << ")" << dendl; + ceph_abort(); + } else { + ldout(cct, 0) << "[!TestPeer] accepted(stale event): " << conn << dendl; + } + flush_pending_send(); + } + bool ms_handle_reset(Connection* conn) override { + if (tracked_conn == conn) { + ldout(cct, 0) << "[TestPeer] reset: " << conn << dendl; + tracked_conn = nullptr; + } else { + ldout(cct, 0) << "[!TestPeer] reset(invalid event): conn(" << conn + << ") != tracked_conn(" << tracked_conn + << ")" << dendl; + } + return true; + } + void ms_handle_remote_reset(Connection* conn) override { + if (tracked_conn == conn) { + ldout(cct, 0) << "[TestPeer] remote reset: " << conn << dendl; + } else { + ldout(cct, 0) << "[!TestPeer] reset(invalid event): conn(" << conn + << ") != tracked_conn(" << tracked_conn + << ")" << dendl; + } + } + bool ms_handle_refused(Connection* conn) override { + ldout(cct, 0) << "[!TestPeer] refused: " << conn << dendl; + return true; + } + + private: + void init(entity_addr_t test_peer_addr, SocketPolicy policy) { + peer_msgr.reset(Messenger::create(cct, "async", entity_name_t::OSD(4), "TestPeer", 4)); + dummy_auth.auth_registry.refresh_config(); + peer_msgr->set_cluster_protocol(CEPH_OSD_PROTOCOL); + peer_msgr->set_default_policy(policy); + peer_msgr->set_auth_client(&dummy_auth); + peer_msgr->set_auth_server(&dummy_auth); + peer_msgr->set_require_authorizer(false); + peer_msgr->bind(test_peer_addr); + peer_msgr->add_dispatcher_head(this); + peer_msgr->start(); + } + + void send_op() { + ceph_assert(tracked_conn); + pg_t pgid; + object_locator_t oloc; + hobject_t hobj(object_t(), oloc.key, CEPH_NOSNAP, pgid.ps(), + pgid.pool(), oloc.nspace); + spg_t spgid(pgid); + tracked_conn->send_message2(make_message<MOSDOp>(0, 0, hobj, spgid, 0, 0, 0)); + } + + void flush_pending_send() { + if (pending_send != 0) { + ldout(cct, 0) << "[TestPeer] flush sending " + << pending_send << " ops" << dendl; + } + ceph_assert(tracked_conn); + while (pending_send) { + send_op(); + --pending_send; + } + } + + public: + FailoverSuitePeer(CephContext* cct, cb_t op_callback) + : Dispatcher(cct), dummy_auth(cct), op_callback(op_callback) { } + + void shutdown() { + peer_msgr->shutdown(); + peer_msgr->wait(); + } + + void connect_peer(entity_addr_t test_addr) { + ldout(cct, 0) << "[TestPeer] connect_peer(" << test_addr << ")" << dendl; + auto conn = peer_msgr->connect_to_osd(entity_addrvec_t{test_addr}); + if (tracked_conn) { + if (tracked_conn == conn.get()) { + ldout(cct, 0) << "[TestPeer] this is not a new session " << conn.get() << dendl; + } else { + ldout(cct, 0) << "[TestPeer] this is a new session " << conn.get() + << ", replacing old one " << tracked_conn << dendl; + } + } else { + ldout(cct, 0) << "[TestPeer] this is a new session " << conn.get() << dendl; + } + tracked_conn = conn.get(); + flush_pending_send(); + } + + void send_peer() { + if (tracked_conn) { + ldout(cct, 0) << "[TestPeer] send_peer()" << dendl; + send_op(); + } else { + ++pending_send; + ldout(cct, 0) << "[TestPeer] send_peer() (pending " << pending_send << ")" << dendl; + } + } + + void keepalive_peer() { + ldout(cct, 0) << "[TestPeer] keepalive_peer()" << dendl; + ceph_assert(tracked_conn); + tracked_conn->send_keepalive(); + } + + void markdown() { + ldout(cct, 0) << "[TestPeer] markdown()" << dendl; + ceph_assert(tracked_conn); + tracked_conn->mark_down(); + tracked_conn = nullptr; + } + + static std::unique_ptr<FailoverSuitePeer> + create(CephContext* cct, entity_addr_t test_peer_addr, + SocketPolicy policy, cb_t op_callback) { + auto suite = std::make_unique<FailoverSuitePeer>(cct, op_callback); + suite->init(test_peer_addr, policy); + return suite; + } +}; + +SocketPolicy to_socket_policy(CephContext* cct, policy_t policy) { + switch (policy) { + case policy_t::stateful_server: + return SocketPolicy::stateful_server(0); + case policy_t::stateless_server: + return SocketPolicy::stateless_server(0); + case policy_t::lossless_peer: + return SocketPolicy::lossless_peer(0); + case policy_t::lossless_peer_reuse: + return SocketPolicy::lossless_peer_reuse(0); + case policy_t::lossy_client: + return SocketPolicy::lossy_client(0); + case policy_t::lossless_client: + return SocketPolicy::lossless_client(0); + default: + lderr(cct) << "[CmdSrv] unexpected policy type" << dendl; + ceph_abort(); + } +} + +class FailoverTestPeer : public Dispatcher { + DummyAuthClientServer dummy_auth; + std::unique_ptr<Messenger> cmd_msgr; + Connection *cmd_conn = nullptr; + const entity_addr_t test_peer_addr; + std::unique_ptr<FailoverSuitePeer> test_suite; + const bool nonstop; + + bool ms_can_fast_dispatch_any() const override { return false; } + bool ms_can_fast_dispatch(const Message* m) const override { return false; } + void ms_fast_dispatch(Message* m) override { ceph_abort(); } + bool ms_dispatch(Message* m) override { + auto conn = m->get_connection().get(); + if (cmd_conn == nullptr) { + ldout(cct, 0) << "[!CmdSrv] got msg from CmdCli(conn " + << conn << "not tracked yet)" << dendl; + cmd_conn = conn; + } else if (cmd_conn != conn) { + lderr(cct) << "[CmdSrv] got msg from CmdCli: conn(" << conn + << ") != cmd_conn(" << cmd_conn + << ")" << dendl; + ceph_abort(); + } else { + // good! + } + switch (m->get_type()) { + case CEPH_MSG_PING: { + ldout(cct, 0) << "[CmdSrv] got PING, sending PONG ..." << dendl; + cmd_conn->send_message2(make_message<MPing>()); + break; + } + case MSG_COMMAND: { + auto m_cmd = boost::static_pointer_cast<MCommand>(m); + auto cmd = static_cast<cmd_t>(m_cmd->cmd[0][0]); + if (cmd == cmd_t::shutdown) { + ldout(cct, 0) << "All tests succeeded" << dendl; + if (!nonstop) { + ldout(cct, 0) << "[CmdSrv] shutdown ..." << dendl; + cmd_msgr->shutdown(); + } else { + ldout(cct, 0) << "[CmdSrv] nonstop set ..." << dendl; + } + } else { + ldout(cct, 0) << "[CmdSrv] got cmd " << cmd << dendl; + handle_cmd(cmd, m_cmd); + ldout(cct, 0) << "[CmdSrv] done, send cmd reply ..." << dendl; + cmd_conn->send_message2(make_message<MCommandReply>()); + } + break; + } + default: + lderr(cct) << "[CmdSrv] " << __func__ << " " << cmd_conn + << " got unexpected msg from CmdCli: " + << m << dendl; + ceph_abort(); + } + m->put(); + return true; + } + void ms_handle_fast_connect(Connection*) override { ceph_abort(); } + void ms_handle_fast_accept(Connection *conn) override { + if (cmd_conn == nullptr) { + ldout(cct, 0) << "[CmdSrv] accepted: " << conn << dendl; + cmd_conn = conn; + } else if (cmd_conn != conn) { + lderr(cct) << "[CmdSrv] accepted: conn(" << conn + << ") != cmd_conn(" << cmd_conn + << ")" << dendl; + ceph_abort(); + } else { + ldout(cct, 0) << "[!CmdSrv] accepted(stale event): " << conn << dendl; + } + } + bool ms_handle_reset(Connection* conn) override { + if (cmd_conn == conn) { + ldout(cct, 0) << "[CmdSrv] reset: " << conn << dendl; + cmd_conn = nullptr; + } else { + ldout(cct, 0) << "[!CmdSrv] reset(invalid event): conn(" << conn + << ") != cmd_conn(" << cmd_conn + << ")" << dendl; + } + return true; + } + void ms_handle_remote_reset(Connection*) override { ceph_abort(); } + bool ms_handle_refused(Connection*) override { ceph_abort(); } + + private: + void notify_recv_op() { + ceph_assert(cmd_conn); + auto m = make_message<MCommand>(); + m->cmd.emplace_back(1, static_cast<char>(cmd_t::suite_recv_op)); + cmd_conn->send_message2(m); + } + + void handle_cmd(cmd_t cmd, MRef<MCommand> m_cmd) { + switch (cmd) { + case cmd_t::suite_start: { + if (test_suite) { + test_suite->shutdown(); + test_suite.reset(); + ldout(cct, 0) << "-------- suite stopped (force) --------\n\n" << dendl; + } + auto p = static_cast<policy_t>(m_cmd->cmd[1][0]); + ldout(cct, 0) << "[CmdSrv] suite starting (" << p + <<", " << test_peer_addr << ") ..." << dendl; + auto policy = to_socket_policy(cct, p); + auto suite = FailoverSuitePeer::create(cct, test_peer_addr, policy, + [this] { notify_recv_op(); }); + test_suite.swap(suite); + return; + } + case cmd_t::suite_stop: + ceph_assert(test_suite); + test_suite->shutdown(); + test_suite.reset(); + ldout(cct, 0) << "-------- suite stopped --------\n\n" << dendl; + return; + case cmd_t::suite_connect_me: { + ceph_assert(test_suite); + entity_addr_t test_addr = entity_addr_t(); + test_addr.parse(m_cmd->cmd[1].c_str(), nullptr); + test_suite->connect_peer(test_addr); + return; + } + case cmd_t::suite_send_me: + ceph_assert(test_suite); + test_suite->send_peer(); + return; + case cmd_t::suite_keepalive_me: + ceph_assert(test_suite); + test_suite->keepalive_peer(); + return; + case cmd_t::suite_markdown: + ceph_assert(test_suite); + test_suite->markdown(); + return; + default: + lderr(cct) << "[CmdSrv] got unexpected command " << m_cmd + << " from CmdCli" << dendl; + ceph_abort(); + } + } + + void init(entity_addr_t cmd_peer_addr) { + cmd_msgr.reset(Messenger::create(cct, "async", entity_name_t::OSD(3), "CmdSrv", 3)); + dummy_auth.auth_registry.refresh_config(); + cmd_msgr->set_cluster_protocol(CEPH_OSD_PROTOCOL); + cmd_msgr->set_default_policy(Messenger::Policy::stateless_server(0)); + cmd_msgr->set_auth_client(&dummy_auth); + cmd_msgr->set_auth_server(&dummy_auth); + cmd_msgr->set_require_authorizer(false); + cmd_msgr->bind(cmd_peer_addr); + cmd_msgr->add_dispatcher_head(this); + cmd_msgr->start(); + } + + public: + FailoverTestPeer(CephContext* cct, + entity_addr_t test_peer_addr, + bool nonstop) + : Dispatcher(cct), + dummy_auth(cct), + test_peer_addr(test_peer_addr), + nonstop(nonstop) { } + + void wait() { cmd_msgr->wait(); } + + static std::unique_ptr<FailoverTestPeer> + create(CephContext* cct, entity_addr_t cmd_peer_addr, bool nonstop) { + // suite bind to cmd_peer_addr, with port + 1 + entity_addr_t test_peer_addr = cmd_peer_addr; + test_peer_addr.set_port(cmd_peer_addr.get_port() + 1); + auto test_peer = std::make_unique<FailoverTestPeer>(cct, test_peer_addr, nonstop); + test_peer->init(cmd_peer_addr); + ldout(cct, 0) << "[CmdSrv] ready" << dendl; + return test_peer; + } +}; + +} + +int main(int argc, char** argv) +{ + namespace po = boost::program_options; + po::options_description desc{"Allowed options"}; + desc.add_options() + ("help,h", "show help message") + ("addr", po::value<std::string>()->default_value("v2:127.0.0.1:9013"), + "CmdSrv address, and TestPeer address with port+=1") + ("nonstop", po::value<bool>()->default_value(false), + "Do not shutdown TestPeer when all tests are successful"); + po::variables_map vm; + std::vector<std::string> unrecognized_options; + try { + auto parsed = po::command_line_parser(argc, argv) + .options(desc) + .allow_unregistered() + .run(); + po::store(parsed, vm); + if (vm.count("help")) { + std::cout << desc << std::endl; + return 0; + } + po::notify(vm); + unrecognized_options = po::collect_unrecognized(parsed.options, po::include_positional); + } catch(const po::error& e) { + std::cerr << "error: " << e.what() << std::endl; + return 1; + } + + auto addr = vm["addr"].as<std::string>(); + entity_addr_t cmd_peer_addr; + cmd_peer_addr.parse(addr.c_str(), nullptr); + auto nonstop = vm["nonstop"].as<bool>(); + + std::vector<const char*> args(argv, argv + argc); + auto cct = global_init(nullptr, args, + CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_MON_CONFIG); + common_init_finish(cct.get()); + cct->_conf.set_val("ms_crc_header", "false"); + cct->_conf.set_val("ms_crc_data", "false"); + + auto test_peer = FailoverTestPeer::create(cct.get(), cmd_peer_addr, nonstop); + test_peer->wait(); +} diff --git a/src/test/crimson/test_monc.cc b/src/test/crimson/test_monc.cc new file mode 100644 index 000000000..f590ce73a --- /dev/null +++ b/src/test/crimson/test_monc.cc @@ -0,0 +1,90 @@ +#include <seastar/core/app-template.hh> +#include "common/ceph_argparse.h" +#include "crimson/common/auth_handler.h" +#include "crimson/common/config_proxy.h" +#include "crimson/mon/MonClient.h" +#include "crimson/net/Connection.h" +#include "crimson/net/Messenger.h" + +using Config = crimson::common::ConfigProxy; +using MonClient = crimson::mon::Client; + +namespace { + +class DummyAuthHandler : public crimson::common::AuthHandler { +public: + void handle_authentication(const EntityName& name, + const AuthCapsInfo& caps) final + {} +}; + +DummyAuthHandler dummy_handler; + +} + +static seastar::future<> test_monc() +{ + return crimson::common::sharded_conf().start(EntityName{}, string_view{"ceph"}).then([] { + std::vector<const char*> args; + std::string cluster; + std::string conf_file_list; + auto init_params = ceph_argparse_early_args(args, + CEPH_ENTITY_TYPE_CLIENT, + &cluster, + &conf_file_list); + auto& conf = crimson::common::local_conf(); + conf->name = init_params.name; + conf->cluster = cluster; + return conf.parse_config_files(conf_file_list); + }).then([] { + return crimson::common::sharded_perf_coll().start(); + }).then([]() mutable { + auto msgr = crimson::net::Messenger::create(entity_name_t::OSD(0), "monc", 0); + auto& conf = crimson::common::local_conf(); + if (conf->ms_crc_data) { + msgr->set_crc_data(); + } + if (conf->ms_crc_header) { + msgr->set_crc_header(); + } + msgr->set_require_authorizer(false); + return seastar::do_with(MonClient{*msgr, dummy_handler}, + [msgr](auto& monc) mutable { + return msgr->start({&monc}).then([&monc] { + return seastar::with_timeout( + seastar::lowres_clock::now() + std::chrono::seconds{10}, + monc.start()); + }).then([&monc] { + return monc.stop(); + }); + }).finally([msgr] { + return msgr->shutdown(); + }); + }).finally([] { + return crimson::common::sharded_perf_coll().stop().then([] { + return crimson::common::sharded_conf().stop(); + }); + }); +} + +int main(int argc, char** argv) +{ + seastar::app_template app; + return app.run(argc, argv, [&] { + return test_monc().then([] { + std::cout << "All tests succeeded" << std::endl; + }).handle_exception([] (auto eptr) { + std::cout << "Test failure" << std::endl; + return seastar::make_exception_future<>(eptr); + }); + }); +} + + +/* + * Local Variables: + * compile-command: "make -j4 \ + * -C ../../../build \ + * unittest_seastar_monc" + * End: + */ diff --git a/src/test/crimson/test_perfcounters.cc b/src/test/crimson/test_perfcounters.cc new file mode 100644 index 000000000..8aecbf911 --- /dev/null +++ b/src/test/crimson/test_perfcounters.cc @@ -0,0 +1,62 @@ +#include <pthread.h> +#include <stdlib.h> +#include <iostream> +#include <fmt/format.h> + +#include "common/Formatter.h" +#include "common/perf_counters.h" +#include "crimson/common/perf_counters_collection.h" + +#include <seastar/core/app-template.hh> +#include <seastar/core/sharded.hh> + +enum { + PERFTEST_FIRST = 1000000, + PERFTEST_INDEX, + PERFTEST_LAST, +}; + +static constexpr uint64_t PERF_VAL = 42; + +static seastar::future<> test_perfcounters(){ + return crimson::common::sharded_perf_coll().start().then([] { + return crimson::common::sharded_perf_coll().invoke_on_all([] (auto& s){ + std::string name =fmt::format("seastar-osd::shard-{}",seastar::this_shard_id()); + PerfCountersBuilder plb(NULL, name, PERFTEST_FIRST,PERFTEST_LAST); + plb.add_u64_counter(PERFTEST_INDEX, "perftest_count", "count perftest"); + auto perf_logger = plb.create_perf_counters(); + perf_logger->inc(PERFTEST_INDEX,PERF_VAL); + s.get_perf_collection()->add(perf_logger); + }); + }).then([]{ + return crimson::common::sharded_perf_coll().invoke_on_all([] (auto& s){ + auto pcc = s.get_perf_collection(); + pcc->with_counters([](auto& by_path){ + for (auto& perf_counter : by_path) { + if (PERF_VAL != perf_counter.second.perf_counters->get(PERFTEST_INDEX)) { + throw std::runtime_error("perf counter does not match"); + } + } + }); + }); + }).finally([] { + return crimson::common::sharded_perf_coll().stop(); + }); + +} + +int main(int argc, char** argv) +{ + seastar::app_template app; + return app.run(argc, argv, [&] { + return test_perfcounters().then([] { + std::cout << "All tests succeeded" << std::endl; + }).handle_exception([] (auto eptr) { + std::cout << "Test failure" << std::endl; + return seastar::make_exception_future<>(eptr); + }); + }); + +} + + diff --git a/src/test/crimson/test_socket.cc b/src/test/crimson/test_socket.cc new file mode 100644 index 000000000..bfdeeea2a --- /dev/null +++ b/src/test/crimson/test_socket.cc @@ -0,0 +1,490 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include <seastar/core/app-template.hh> +#include <seastar/core/gate.hh> +#include <seastar/core/sharded.hh> +#include <seastar/core/sleep.hh> +#include <seastar/core/when_all.hh> +#include <seastar/util/later.hh> + +#include "crimson/common/log.h" +#include "crimson/net/Errors.h" +#include "crimson/net/Fwd.h" +#include "crimson/net/Socket.h" + +namespace { + +using seastar::engine; +using seastar::future; +using crimson::net::error; +using crimson::net::FixedCPUServerSocket; +using crimson::net::Socket; +using crimson::net::SocketRef; +using crimson::net::stop_t; + +using SocketFRef = seastar::foreign_ptr<SocketRef>; + +static seastar::logger logger{"crimsontest"}; +static entity_addr_t get_server_addr() { + static int port = 9020; + ++port; + ceph_assert(port < 9030 && "socket and messenger test ports should not overlap"); + entity_addr_t saddr; + saddr.parse("127.0.0.1", nullptr); + saddr.set_port(port); + return saddr; +} + +future<SocketRef> socket_connect(const entity_addr_t& saddr) { + logger.debug("socket_connect() to {} ...", saddr); + return Socket::connect(saddr).then([] (auto socket) { + logger.debug("socket_connect() connected"); + return socket; + }); +} + +future<> test_refused() { + logger.info("test_refused()..."); + auto saddr = get_server_addr(); + return socket_connect(saddr).discard_result().then([saddr] { + logger.error("test_refused(): connection to {} is not refused", saddr); + ceph_abort(); + }).handle_exception_type([] (const std::system_error& e) { + if (e.code() != std::errc::connection_refused) { + logger.error("test_refused() got unexpeted error {}", e); + ceph_abort(); + } else { + logger.info("test_refused() ok\n"); + } + }).handle_exception([] (auto eptr) { + logger.error("test_refused() got unexpeted exception {}", eptr); + ceph_abort(); + }); +} + +future<> test_bind_same() { + logger.info("test_bind_same()..."); + return FixedCPUServerSocket::create().then([] (auto pss1) { + auto saddr = get_server_addr(); + return pss1->listen(saddr).safe_then([saddr] { + // try to bind the same address + return FixedCPUServerSocket::create().then([saddr] (auto pss2) { + return pss2->listen(saddr).safe_then([] { + logger.error("test_bind_same() should raise address_in_use"); + ceph_abort(); + }, FixedCPUServerSocket::listen_ertr::all_same_way( + [] (const std::error_code& e) { + if (e == std::errc::address_in_use) { + // successful! + logger.info("test_bind_same() ok\n"); + } else { + logger.error("test_bind_same() got unexpected error {}", e); + ceph_abort(); + } + // Note: need to return a explicit ready future, or there will be a + // runtime error: member access within null pointer of type 'struct promise_base' + return seastar::now(); + })).then([pss2] { + return pss2->destroy(); + }); + }); + }, FixedCPUServerSocket::listen_ertr::all_same_way( + [saddr] (const std::error_code& e) { + logger.error("test_bind_same(): there is another instance running at {}", + saddr); + ceph_abort(); + })).then([pss1] { + return pss1->destroy(); + }).handle_exception([] (auto eptr) { + logger.error("test_bind_same() got unexpeted exception {}", eptr); + ceph_abort(); + }); + }); +} + +future<> test_accept() { + logger.info("test_accept()"); + return FixedCPUServerSocket::create().then([] (auto pss) { + auto saddr = get_server_addr(); + return pss->listen(saddr).safe_then([pss] { + return pss->accept([] (auto socket, auto paddr) { + // simple accept + return seastar::sleep(100ms).then([socket = std::move(socket)] () mutable { + return socket->close().finally([cleanup = std::move(socket)] {}); + }); + }); + }, FixedCPUServerSocket::listen_ertr::all_same_way( + [saddr] (const std::error_code& e) { + logger.error("test_accept(): there is another instance running at {}", + saddr); + ceph_abort(); + })).then([saddr] { + return seastar::when_all( + socket_connect(saddr).then([] (auto socket) { + return socket->close().finally([cleanup = std::move(socket)] {}); }), + socket_connect(saddr).then([] (auto socket) { + return socket->close().finally([cleanup = std::move(socket)] {}); }), + socket_connect(saddr).then([] (auto socket) { + return socket->close().finally([cleanup = std::move(socket)] {}); }) + ).discard_result(); + }).then([] { + // should be enough to be connected locally + return seastar::sleep(50ms); + }).then([] { + logger.info("test_accept() ok\n"); + }).then([pss] { + return pss->destroy(); + }).handle_exception([] (auto eptr) { + logger.error("test_accept() got unexpeted exception {}", eptr); + ceph_abort(); + }); + }); +} + +class SocketFactory { + SocketRef client_socket; + SocketFRef server_socket; + FixedCPUServerSocket *pss = nullptr; + seastar::promise<> server_connected; + + public: + // cb_client() on CPU#0, cb_server() on CPU#1 + template <typename FuncC, typename FuncS> + static future<> dispatch_sockets(FuncC&& cb_client, FuncS&& cb_server) { + assert(seastar::this_shard_id() == 0u); + auto owner = std::make_unique<SocketFactory>(); + auto psf = owner.get(); + auto saddr = get_server_addr(); + return seastar::smp::submit_to(1u, [psf, saddr] { + return FixedCPUServerSocket::create().then([psf, saddr] (auto pss) { + psf->pss = pss; + return pss->listen(saddr + ).safe_then([]{}, FixedCPUServerSocket::listen_ertr::all_same_way( + [saddr] (const std::error_code& e) { + logger.error("dispatch_sockets(): there is another instance running at {}", + saddr); + ceph_abort(); + })); + }); + }).then([psf, saddr] { + return seastar::when_all_succeed( + seastar::smp::submit_to(0u, [psf, saddr] { + return socket_connect(saddr).then([psf] (auto socket) { + psf->client_socket = std::move(socket); + }); + }), + seastar::smp::submit_to(1u, [psf] { + return psf->pss->accept([psf] (auto socket, auto paddr) { + psf->server_socket = seastar::make_foreign(std::move(socket)); + return seastar::smp::submit_to(0u, [psf] { + psf->server_connected.set_value(); + }); + }); + }) + ); + }).then_unpack([] { + return seastar::now(); + }).then([psf] { + return psf->server_connected.get_future(); + }).then([psf] { + if (psf->pss) { + return seastar::smp::submit_to(1u, [psf] { + return psf->pss->destroy(); + }); + } + return seastar::now(); + }).then([psf, + cb_client = std::move(cb_client), + cb_server = std::move(cb_server)] () mutable { + logger.debug("dispatch_sockets(): client/server socket are ready"); + return seastar::when_all_succeed( + seastar::smp::submit_to(0u, [socket = psf->client_socket.get(), + cb_client = std::move(cb_client)] { + return cb_client(socket).then([socket] { + logger.debug("closing client socket..."); + return socket->close(); + }).handle_exception([] (auto eptr) { + logger.error("dispatch_sockets():" + " cb_client() got unexpeted exception {}", eptr); + ceph_abort(); + }); + }), + seastar::smp::submit_to(1u, [socket = psf->server_socket.get(), + cb_server = std::move(cb_server)] { + return cb_server(socket).then([socket] { + logger.debug("closing server socket..."); + return socket->close(); + }).handle_exception([] (auto eptr) { + logger.error("dispatch_sockets():" + " cb_server() got unexpeted exception {}", eptr); + ceph_abort(); + }); + }) + ); + }).then_unpack([] { + return seastar::now(); + }).finally([cleanup = std::move(owner)] {}); + } +}; + +class Connection { + static const uint64_t DATA_TAIL = 5327; + static const unsigned DATA_SIZE = 4096; + std::array<uint64_t, DATA_SIZE> data = {0}; + + void verify_data_read(const uint64_t read_data[]) { + ceph_assert(read_data[0] == read_count); + ceph_assert(data[DATA_SIZE - 1] = DATA_TAIL); + } + + Socket* socket = nullptr; + uint64_t write_count = 0; + uint64_t read_count = 0; + + Connection(Socket* socket) : socket{socket} { + assert(socket); + data[DATA_SIZE - 1] = DATA_TAIL; + } + + future<> dispatch_write(unsigned round = 0, bool force_shut = false) { + logger.debug("dispatch_write(round={}, force_shut={})...", round, force_shut); + return seastar::repeat([this, round, force_shut] { + if (round != 0 && round <= write_count) { + return seastar::futurize_invoke([this, force_shut] { + if (force_shut) { + logger.debug("dispatch_write() done, force shutdown output"); + socket->force_shutdown_out(); + } else { + logger.debug("dispatch_write() done"); + } + }).then([] { + return seastar::make_ready_future<stop_t>(stop_t::yes); + }); + } else { + data[0] = write_count; + return socket->write(seastar::net::packet( + reinterpret_cast<const char*>(&data), sizeof(data)) + ).then([this] { + return socket->flush(); + }).then([this] { + write_count += 1; + return seastar::make_ready_future<stop_t>(stop_t::no); + }); + } + }); + } + + future<> dispatch_write_unbounded() { + return dispatch_write( + ).then([] { + ceph_abort(); + }).handle_exception_type([this] (const std::system_error& e) { + if (e.code() != std::errc::broken_pipe && + e.code() != std::errc::connection_reset) { + logger.error("dispatch_write_unbounded(): " + "unexpected error {}", e); + throw; + } + // successful + logger.debug("dispatch_write_unbounded(): " + "expected error {}", e); + shutdown(); + }); + } + + future<> dispatch_read(unsigned round = 0, bool force_shut = false) { + logger.debug("dispatch_read(round={}, force_shut={})...", round, force_shut); + return seastar::repeat([this, round, force_shut] { + if (round != 0 && round <= read_count) { + return seastar::futurize_invoke([this, force_shut] { + if (force_shut) { + logger.debug("dispatch_read() done, force shutdown input"); + socket->force_shutdown_in(); + } else { + logger.debug("dispatch_read() done"); + } + }).then([] { + return seastar::make_ready_future<stop_t>(stop_t::yes); + }); + } else { + return seastar::futurize_invoke([this] { + // we want to test both Socket::read() and Socket::read_exactly() + if (read_count % 2) { + return socket->read(DATA_SIZE * sizeof(uint64_t) + ).then([this] (ceph::bufferlist bl) { + uint64_t read_data[DATA_SIZE]; + auto p = bl.cbegin(); + ::ceph::decode_raw(read_data, p); + verify_data_read(read_data); + }); + } else { + return socket->read_exactly(DATA_SIZE * sizeof(uint64_t) + ).then([this] (auto buf) { + auto read_data = reinterpret_cast<const uint64_t*>(buf.get()); + verify_data_read(read_data); + }); + } + }).then([this] { + ++read_count; + return seastar::make_ready_future<stop_t>(stop_t::no); + }); + } + }); + } + + future<> dispatch_read_unbounded() { + return dispatch_read( + ).then([] { + ceph_abort(); + }).handle_exception_type([this] (const std::system_error& e) { + if (e.code() != error::read_eof + && e.code() != std::errc::connection_reset) { + logger.error("dispatch_read_unbounded(): " + "unexpected error {}", e); + throw; + } + // successful + logger.debug("dispatch_read_unbounded(): " + "expected error {}", e); + shutdown(); + }); + } + + void shutdown() { + socket->shutdown(); + } + + public: + static future<> dispatch_rw_bounded(Socket* socket, unsigned round, + bool force_shut = false) { + logger.debug("dispatch_rw_bounded(round={}, force_shut={})...", + round, force_shut); + return seastar::do_with(Connection{socket}, + [round, force_shut] (auto& conn) { + ceph_assert(round != 0); + return seastar::when_all_succeed( + conn.dispatch_write(round, force_shut), + conn.dispatch_read(round, force_shut) + ).then_unpack([] { + return seastar::now(); + }); + }); + } + + static future<> dispatch_rw_unbounded(Socket* socket, bool preemptive_shut = false) { + logger.debug("dispatch_rw_unbounded(preemptive_shut={})...", preemptive_shut); + return seastar::do_with(Connection{socket}, [preemptive_shut] (auto& conn) { + return seastar::when_all_succeed( + conn.dispatch_write_unbounded(), + conn.dispatch_read_unbounded(), + seastar::futurize_invoke([&conn, preemptive_shut] { + if (preemptive_shut) { + return seastar::sleep(100ms).then([&conn] { + logger.debug("dispatch_rw_unbounded() shutdown socket preemptively(100ms)"); + conn.shutdown(); + }); + } else { + return seastar::now(); + } + }) + ).then_unpack([] { + return seastar::now(); + }); + }); + } +}; + +future<> test_read_write() { + logger.info("test_read_write()..."); + return SocketFactory::dispatch_sockets( + [] (auto cs) { return Connection::dispatch_rw_bounded(cs, 128); }, + [] (auto ss) { return Connection::dispatch_rw_bounded(ss, 128); } + ).then([] { + logger.info("test_read_write() ok\n"); + }).handle_exception([] (auto eptr) { + logger.error("test_read_write() got unexpeted exception {}", eptr); + ceph_abort(); + }); +} + +future<> test_unexpected_down() { + logger.info("test_unexpected_down()..."); + return SocketFactory::dispatch_sockets( + [] (auto cs) { + return Connection::dispatch_rw_bounded(cs, 128, true + ).handle_exception_type([] (const std::system_error& e) { + logger.debug("test_unexpected_down(): client get error {}", e); + ceph_assert(e.code() == error::read_eof); + }); + }, + [] (auto ss) { return Connection::dispatch_rw_unbounded(ss); } + ).then([] { + logger.info("test_unexpected_down() ok\n"); + }).handle_exception([] (auto eptr) { + logger.error("test_unexpected_down() got unexpeted exception {}", eptr); + ceph_abort(); + }); +} + +future<> test_shutdown_propagated() { + logger.info("test_shutdown_propagated()..."); + return SocketFactory::dispatch_sockets( + [] (auto cs) { + logger.debug("test_shutdown_propagated() shutdown client socket"); + cs->shutdown(); + return seastar::now(); + }, + [] (auto ss) { return Connection::dispatch_rw_unbounded(ss); } + ).then([] { + logger.info("test_shutdown_propagated() ok\n"); + }).handle_exception([] (auto eptr) { + logger.error("test_shutdown_propagated() got unexpeted exception {}", eptr); + ceph_abort(); + }); +} + +future<> test_preemptive_down() { + logger.info("test_preemptive_down()..."); + return SocketFactory::dispatch_sockets( + [] (auto cs) { return Connection::dispatch_rw_unbounded(cs, true); }, + [] (auto ss) { return Connection::dispatch_rw_unbounded(ss); } + ).then([] { + logger.info("test_preemptive_down() ok\n"); + }).handle_exception([] (auto eptr) { + logger.error("test_preemptive_down() got unexpeted exception {}", eptr); + ceph_abort(); + }); +} + +} + +int main(int argc, char** argv) +{ + seastar::app_template app; + return app.run(argc, argv, [] { + return seastar::futurize_invoke([] { + return test_refused(); + }).then([] { + return test_bind_same(); + }).then([] { + return test_accept(); + }).then([] { + return test_read_write(); + }).then([] { + return test_unexpected_down(); + }).then([] { + return test_shutdown_propagated(); + }).then([] { + return test_preemptive_down(); + }).then([] { + logger.info("All tests succeeded"); + // Seastar has bugs to have events undispatched during shutdown, + // which will result in memory leak and thus fail LeakSanitizer. + return seastar::sleep(100ms); + }).handle_exception([] (auto eptr) { + std::cout << "Test failure" << std::endl; + return seastar::make_exception_future<>(eptr); + }); + }); +} |