diff options
Diffstat (limited to 'src/test/rbd_mirror')
50 files changed, 26930 insertions, 0 deletions
diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt new file mode 100644 index 000000000..1226735d5 --- /dev/null +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -0,0 +1,106 @@ +set(rbd_mirror_test_srcs + test_ClusterWatcher.cc + test_PoolWatcher.cc + test_ImageDeleter.cc + test_ImageReplayer.cc + test_ImageSync.cc + test_InstanceWatcher.cc + test_Instances.cc + test_LeaderWatcher.cc + test_fixture.cc + image_map/test_Policy.cc + ) +add_library(rbd_mirror_test STATIC ${rbd_mirror_test_srcs}) +target_link_libraries(rbd_mirror_test + rbd_test_support + GTest::GTest) + +add_executable(unittest_rbd_mirror + test_main.cc + test_mock_fixture.cc + test_mock_ImageMap.cc + test_mock_ImageReplayer.cc + test_mock_ImageSync.cc + test_mock_InstanceReplayer.cc + test_mock_InstanceWatcher.cc + test_mock_LeaderWatcher.cc + test_mock_MirrorStatusUpdater.cc + test_mock_NamespaceReplayer.cc + test_mock_PoolReplayer.cc + test_mock_PoolWatcher.cc + test_mock_Throttler.cc + image_deleter/test_mock_SnapshotPurgeRequest.cc + image_deleter/test_mock_TrashMoveRequest.cc + image_deleter/test_mock_TrashRemoveRequest.cc + image_deleter/test_mock_TrashWatcher.cc + image_replayer/test_mock_BootstrapRequest.cc + image_replayer/test_mock_CreateImageRequest.cc + image_replayer/test_mock_GetMirrorImageIdRequest.cc + image_replayer/test_mock_PrepareLocalImageRequest.cc + image_replayer/test_mock_PrepareRemoteImageRequest.cc + image_replayer/journal/test_mock_CreateLocalImageRequest.cc + image_replayer/journal/test_mock_PrepareReplayRequest.cc + image_replayer/journal/test_mock_EventPreprocessor.cc + image_replayer/journal/test_mock_Replayer.cc + image_replayer/snapshot/test_mock_ApplyImageStateRequest.cc + image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc + image_replayer/snapshot/test_mock_Replayer.cc + image_sync/test_mock_SyncPointCreateRequest.cc + image_sync/test_mock_SyncPointPruneRequest.cc + pool_watcher/test_mock_RefreshImagesRequest.cc + ) +add_ceph_unittest(unittest_rbd_mirror) + +add_dependencies(unittest_rbd_mirror + cls_journal + cls_lock + cls_rbd) +target_link_libraries(unittest_rbd_mirror + rbd_mirror_test + rbd_mirror_internal + rbd_mirror_types + rbd_api + rbd_internal + rbd_test_mock + journal + journal_test_mock + cls_rbd_client + cls_lock_client + cls_journal_client + rbd_types + rados_test_stub + librados + osdc + global + radostest-cxx + ) + +add_executable(ceph_test_rbd_mirror + test_main.cc + ) + +target_link_libraries(ceph_test_rbd_mirror + rbd_mirror_test + rbd_mirror_internal + rbd_mirror_types + rbd_api + rbd_internal + journal + cls_rbd_client + cls_journal_client + rbd_types + libneorados + librados + radostest-cxx + ${UNITTEST_LIBS} + ) + +add_executable(ceph_test_rbd_mirror_random_write + random_write.cc) +target_link_libraries(ceph_test_rbd_mirror_random_write + librbd librados global) + +install(TARGETS + ceph_test_rbd_mirror + ceph_test_rbd_mirror_random_write + DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/test/rbd_mirror/image_deleter/test_mock_SnapshotPurgeRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_SnapshotPurgeRequest.cc new file mode 100644 index 000000000..2f14854de --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_SnapshotPurgeRequest.cc @@ -0,0 +1,430 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockExclusiveLock.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockImageState.h" +#include "test/librbd/mock/MockOperations.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + static MockTestImageCtx *s_instance; + static MockTestImageCtx *create(const std::string &image_name, + const std::string &image_id, + const char *snap, librados::IoCtx& p, + bool read_only) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + s_instance = this; + } +}; + +MockTestImageCtx *MockTestImageCtx::s_instance = nullptr; + +} // anonymous namespace +} // namespace librbd + +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::InSequence; +using ::testing::WithArg; + +class TestMockImageDeleterSnapshotPurgeRequest : public TestMockFixture { +public: + typedef SnapshotPurgeRequest<librbd::MockTestImageCtx> MockSnapshotPurgeRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_set_journal_policy(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, set_journal_policy(_)) + .WillOnce(Invoke([](librbd::journal::Policy* policy) { + delete policy; + })); + } + + void expect_open(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, open(true, _)) + .WillOnce(WithArg<1>(Invoke([this, &mock_image_ctx, r](Context* ctx) { + EXPECT_EQ(0U, mock_image_ctx.read_only_mask & + librbd::IMAGE_READ_ONLY_FLAG_NON_PRIMARY); + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_close(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, close(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_acquire_lock(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, acquire_lock(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_get_snap_namespace(librbd::MockTestImageCtx &mock_image_ctx, + uint64_t snap_id, + const cls::rbd::SnapshotNamespace &snap_namespace, + int r) { + EXPECT_CALL(mock_image_ctx, get_snap_namespace(snap_id, _)) + .WillOnce(WithArg<1>(Invoke([snap_namespace, r](cls::rbd::SnapshotNamespace *ns) { + *ns = snap_namespace; + return r; + }))); + } + + void expect_get_snap_name(librbd::MockTestImageCtx &mock_image_ctx, + uint64_t snap_id, const std::string& name, + int r) { + EXPECT_CALL(mock_image_ctx, get_snap_name(snap_id, _)) + .WillOnce(WithArg<1>(Invoke([name, r](std::string *n) { + *n = name; + return r; + }))); + } + + void expect_is_snap_protected(librbd::MockTestImageCtx &mock_image_ctx, + uint64_t snap_id, bool is_protected, int r) { + EXPECT_CALL(mock_image_ctx, is_snap_protected(snap_id, _)) + .WillOnce(WithArg<1>(Invoke([is_protected, r](bool *prot) { + *prot = is_protected; + return r; + }))); + } + + void expect_snap_unprotect(librbd::MockTestImageCtx &mock_image_ctx, + const cls::rbd::SnapshotNamespace& ns, + const std::string& name, int r) { + EXPECT_CALL(*mock_image_ctx.operations, execute_snap_unprotect(ns, name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_snap_remove(librbd::MockTestImageCtx &mock_image_ctx, + const cls::rbd::SnapshotNamespace& ns, + const std::string& name, int r) { + EXPECT_CALL(*mock_image_ctx.operations, execute_snap_remove(ns, name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_start_op(librbd::MockTestImageCtx &mock_image_ctx, bool success) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, start_op(_)) + .WillOnce(Invoke([success](int* r) { + auto f = [](int r) {}; + if (!success) { + *r = -EROFS; + return static_cast<LambdaContext<decltype(f)>*>(nullptr); + } + return new LambdaContext(std::move(f)); + })); + } + + librbd::ImageCtx *m_local_image_ctx; +}; + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SuccessJournal) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap2", 2, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 2, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 2, "snap2", 0); + expect_is_snap_protected(mock_image_ctx, 2, false, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap2", + 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, true, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_unprotect(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, + "snap1", 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap1", + 0); + + expect_close(mock_image_ctx, 0); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SuccessSnapshot) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap2", 2, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 2, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 2, "snap2", 0); + expect_is_snap_protected(mock_image_ctx, 2, false, 0); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap2", + 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, true, 0); + expect_snap_unprotect(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, + "snap1", 0); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap1", + 0); + + expect_close(mock_image_ctx, 0); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, OpenError) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, -EPERM); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, AcquireLockError) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, -EPERM); + expect_close(mock_image_ctx, -EINVAL); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SnapUnprotectBusy) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, true, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_unprotect(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, + "snap1", -EBUSY); + + expect_close(mock_image_ctx, -EINVAL); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EBUSY, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SnapUnprotectError) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, true, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_unprotect(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, + "snap1", -EPERM); + + expect_close(mock_image_ctx, -EINVAL); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SnapRemoveError) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, false, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap1", + -EINVAL); + + expect_close(mock_image_ctx, -EPERM); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, CloseError) { + { + std::unique_lock image_locker{m_local_image_ctx->image_lock}; + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, false, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap1", + 0); + + expect_close(mock_image_ctx, -EINVAL); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc new file mode 100644 index 000000000..e1a52c876 --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc @@ -0,0 +1,901 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/TrashWatcher.h" +#include "librbd/journal/ResetRequest.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "librbd/mirror/ImageRemoveRequest.h" +#include "librbd/trash/MoveRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockExclusiveLock.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockImageState.h" +#include "test/librbd/mock/MockOperations.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + static MockTestImageCtx *s_instance; + static MockTestImageCtx *create(const std::string &image_name, + const std::string &image_id, + const char *snap, librados::IoCtx& p, + bool read_only) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + s_instance = this; + } +}; + +MockTestImageCtx *MockTestImageCtx::s_instance = nullptr; + +} // anonymous namespace + +template<> +struct TrashWatcher<MockTestImageCtx> { + static TrashWatcher* s_instance; + static void notify_image_added(librados::IoCtx&, const std::string& image_id, + const cls::rbd::TrashImageSpec& spec, + Context *ctx) { + ceph_assert(s_instance != nullptr); + s_instance->notify_image_added(image_id, spec, ctx); + } + + MOCK_METHOD3(notify_image_added, void(const std::string&, + const cls::rbd::TrashImageSpec&, + Context*)); + + TrashWatcher() { + s_instance = this; + } +}; + +TrashWatcher<MockTestImageCtx>* TrashWatcher<MockTestImageCtx>::s_instance = nullptr; + +namespace journal { + +template <> +struct ResetRequest<MockTestImageCtx> { + static ResetRequest* s_instance; + Context* on_finish = nullptr; + + static ResetRequest* create(librados::IoCtx &io_ctx, + const std::string &image_id, + const std::string &client_id, + const std::string &mirror_uuid, + ContextWQ *op_work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_EQ(librbd::Journal<>::LOCAL_MIRROR_UUID, mirror_uuid); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + ResetRequest() { + s_instance = this; + } +}; + +ResetRequest<MockTestImageCtx>* ResetRequest<MockTestImageCtx>::s_instance = nullptr; + +} // namespace journal + +namespace mirror { + +template<> +struct GetInfoRequest<librbd::MockTestImageCtx> { + static GetInfoRequest* s_instance; + cls::rbd::MirrorImage *mirror_image; + PromotionState *promotion_state; + std::string *primary_mirror_uuid; + Context *on_finish = nullptr; + + static GetInfoRequest* create(librados::IoCtx& io_ctx, + librbd::asio::ContextWQ* context_wq, + const std::string& image_id, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->mirror_image = mirror_image; + s_instance->promotion_state = promotion_state; + s_instance->primary_mirror_uuid = primary_mirror_uuid; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetInfoRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~GetInfoRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +GetInfoRequest<librbd::MockTestImageCtx>* GetInfoRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct ImageRemoveRequest<librbd::MockTestImageCtx> { + static ImageRemoveRequest* s_instance; + std::string global_image_id; + std::string image_id; + Context* on_finish; + + static ImageRemoveRequest *create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + const std::string& image_id, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->global_image_id = global_image_id; + s_instance->image_id = image_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + ImageRemoveRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~ImageRemoveRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +ImageRemoveRequest<librbd::MockTestImageCtx>* ImageRemoveRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +namespace trash { + +template <> +struct MoveRequest<MockTestImageCtx> { + static MoveRequest* s_instance; + Context* on_finish = nullptr; + + typedef boost::optional<utime_t> DefermentEndTime; + + static MoveRequest* create(librados::IoCtx& io_ctx, + const std::string& image_id, + const cls::rbd::TrashImageSpec& trash_image_spec, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->construct(image_id, trash_image_spec); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD2(construct, void(const std::string&, + const cls::rbd::TrashImageSpec&)); + MOCK_METHOD0(send, void()); + + MoveRequest() { + s_instance = this; + } +}; + +MoveRequest<MockTestImageCtx>* MoveRequest<MockTestImageCtx>::s_instance = nullptr; + +} // namespace trash +} // namespace librbd + +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageDeleterTrashMoveRequest : public TestMockFixture { +public: + typedef TrashMoveRequest<librbd::MockTestImageCtx> MockTrashMoveRequest; + typedef librbd::journal::ResetRequest<librbd::MockTestImageCtx> MockJournalResetRequest; + typedef librbd::mirror::GetInfoRequest<librbd::MockTestImageCtx> MockGetMirrorInfoRequest; + typedef librbd::mirror::ImageRemoveRequest<librbd::MockTestImageCtx> MockImageRemoveRequest; + typedef librbd::trash::MoveRequest<librbd::MockTestImageCtx> MockLibrbdTrashMoveRequest; + typedef librbd::TrashWatcher<librbd::MockTestImageCtx> MockTrashWatcher; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_mirror_image_get_image_id(const std::string& image_id, int r) { + bufferlist bl; + encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_get_image_id"), _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_get_mirror_info( + MockGetMirrorInfoRequest &mock_get_mirror_info_request, + const cls::rbd::MirrorImage &mirror_image, + librbd::mirror::PromotionState promotion_state, + const std::string& primary_mirror_uuid, int r) { + EXPECT_CALL(mock_get_mirror_info_request, send()) + .WillOnce(Invoke([this, &mock_get_mirror_info_request, mirror_image, + promotion_state, primary_mirror_uuid, r]() { + *mock_get_mirror_info_request.mirror_image = mirror_image; + *mock_get_mirror_info_request.promotion_state = promotion_state; + *mock_get_mirror_info_request.primary_mirror_uuid = + primary_mirror_uuid; + m_threads->work_queue->queue( + mock_get_mirror_info_request.on_finish, r); + })); + } + + void expect_set_journal_policy(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, set_journal_policy(_)) + .WillOnce(Invoke([](librbd::journal::Policy* policy) { + delete policy; + })); + } + + void expect_open(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, open(true, _)) + .WillOnce(WithArg<1>(Invoke([this, &mock_image_ctx, r](Context* ctx) { + EXPECT_EQ(0U, mock_image_ctx.read_only_mask & + librbd::IMAGE_READ_ONLY_FLAG_NON_PRIMARY); + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_close(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, close(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_block_requests(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, block_requests(0)).Times(1); + } + + void expect_acquire_lock(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, acquire_lock(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_mirror_image_set(const std::string& image_id, + const cls::rbd::MirrorImage& mirror_image, + int r) { + bufferlist bl; + encode(image_id, bl); + encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_set"), ContentsEqual(bl), _, _, _)) + .WillOnce(Return(r)); + } + + void expect_mirror_image_remove_request( + MockImageRemoveRequest& mock_image_remove_request, int r) { + EXPECT_CALL(mock_image_remove_request, send()) + .WillOnce(Invoke([this, &mock_image_remove_request, r]() { + m_threads->work_queue->queue(mock_image_remove_request.on_finish, r); + })); + } + + void expect_journal_reset(MockJournalResetRequest& mock_journal_reset_request, + int r) { + EXPECT_CALL(mock_journal_reset_request, send()) + .WillOnce(Invoke([this, &mock_journal_reset_request, r]() { + m_threads->work_queue->queue(mock_journal_reset_request.on_finish, r); + })); + } + + void expect_trash_move(MockLibrbdTrashMoveRequest& mock_trash_move_request, + const std::string& image_name, + const std::string& image_id, + const boost::optional<uint32_t>& delay, int r) { + EXPECT_CALL(mock_trash_move_request, construct(image_id, _)) + .WillOnce(WithArg<1>(Invoke([image_name, delay](const cls::rbd::TrashImageSpec& spec) { + ASSERT_EQ(cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, + spec.source); + ASSERT_EQ(image_name, spec.name); + if (delay) { + utime_t time{spec.deletion_time}; + time += *delay; + ASSERT_TRUE(time == spec.deferment_end_time); + } else { + ASSERT_EQ(spec.deletion_time, spec.deferment_end_time); + } + }))); + EXPECT_CALL(mock_trash_move_request, send()) + .WillOnce(Invoke([this, &mock_trash_move_request, r]() { + m_threads->work_queue->queue(mock_trash_move_request.on_finish, r); + })); + } + + void expect_notify_image_added(MockTrashWatcher& mock_trash_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_trash_watcher, notify_image_added(image_id, _, _)) + .WillOnce(WithArg<2>(Invoke([this](Context *ctx) { + m_threads->work_queue->queue(ctx, 0); + }))); + } + + librbd::ImageCtx *m_local_image_ctx; +}; + +TEST_F(TestMockImageDeleterTrashMoveRequest, SuccessJournal) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); + + expect_close(mock_image_ctx, 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, SuccessSnapshot) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); + + expect_close(mock_image_ctx, 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + false, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetImageIdDNE) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -ENOENT); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetImageIdError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetMirrorInfoLocalPrimary) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetMirrorInfoOrphan) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + false, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetMirrorInfoDNE) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", -ENOENT); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetMirrorInfoError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, DisableMirrorImageError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, OpenImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, ResetJournalError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, -EINVAL); + + expect_close(mock_image_ctx, 0); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, AcquireLockError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, -EINVAL); + + expect_close(mock_image_ctx, 0); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, TrashMoveError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, -EINVAL); + + expect_close(mock_image_ctx, 0); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, RemoveMirrorImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, -EINVAL); + + expect_close(mock_image_ctx, 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, CloseImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_ORPHAN, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); + + expect_close(mock_image_ctx, -EINVAL); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, DelayedDelation) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.config.set_val("rbd_mirroring_delete_delay", "600"); + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + expect_mirror_image_set("image id", + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + 600, 0); + + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); + expect_close(mock_image_ctx, 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashRemoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashRemoveRequest.cc new file mode 100644 index 000000000..f69a74f64 --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashRemoveRequest.cc @@ -0,0 +1,453 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/TrashWatcher.h" +#include "librbd/Utils.h" +#include "librbd/trash/RemoveRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h" +#include "tools/rbd_mirror/image_deleter/TrashRemoveRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +template<> +struct TrashWatcher<MockTestImageCtx> { + static TrashWatcher* s_instance; + static void notify_image_removed(librados::IoCtx&, + const std::string& image_id, Context *ctx) { + ceph_assert(s_instance != nullptr); + s_instance->notify_image_removed(image_id, ctx); + } + + MOCK_METHOD2(notify_image_removed, void(const std::string&, Context*)); + + TrashWatcher() { + s_instance = this; + } +}; + +TrashWatcher<MockTestImageCtx>* TrashWatcher<MockTestImageCtx>::s_instance = nullptr; + +namespace trash { + +template <> +struct RemoveRequest<librbd::MockTestImageCtx> { + static RemoveRequest *s_instance; + Context *on_finish = nullptr; + + static RemoveRequest *create(librados::IoCtx &io_ctx, + const std::string &image_id, + librbd::asio::ContextWQ *work_queue, + bool force, + librbd::ProgressContext &progress_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_TRUE(force); + s_instance->construct(image_id); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); + + RemoveRequest() { + s_instance = this; + } +}; + +RemoveRequest<librbd::MockTestImageCtx>* RemoveRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace trash +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_deleter { + +template <> +struct SnapshotPurgeRequest<librbd::MockTestImageCtx> { + static SnapshotPurgeRequest *s_instance; + Context *on_finish = nullptr; + + static SnapshotPurgeRequest *create(librados::IoCtx &io_ctx, + const std::string &image_id, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->construct(image_id); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); + + SnapshotPurgeRequest() { + s_instance = this; + } +}; + +SnapshotPurgeRequest<librbd::MockTestImageCtx>* SnapshotPurgeRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_deleter/TrashRemoveRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::Invoke; +using ::testing::InSequence; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageDeleterTrashRemoveRequest : public TestMockFixture { +public: + typedef TrashRemoveRequest<librbd::MockTestImageCtx> MockTrashRemoveRequest; + typedef SnapshotPurgeRequest<librbd::MockTestImageCtx> MockSnapshotPurgeRequest; + typedef librbd::TrashWatcher<librbd::MockTestImageCtx> MockTrashWatcher; + typedef librbd::trash::RemoveRequest<librbd::MockTestImageCtx> MockLibrbdTrashRemoveRequest; + + void expect_trash_get(const cls::rbd::TrashImageSpec& trash_spec, int r) { + using ceph::encode; + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(StrEq(RBD_TRASH), _, StrEq("rbd"), + StrEq("trash_get"), _, _, _, _)) + .WillOnce(WithArg<5>(Invoke([trash_spec, r](bufferlist* bl) { + encode(trash_spec, *bl); + return r; + }))); + } + + void expect_trash_state_set(const std::string& image_id, int r) { + bufferlist in_bl; + encode(image_id, in_bl); + encode(cls::rbd::TRASH_IMAGE_STATE_REMOVING, in_bl); + encode(cls::rbd::TRASH_IMAGE_STATE_NORMAL, in_bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(StrEq(RBD_TRASH), _, StrEq("rbd"), + StrEq("trash_state_set"), + ContentsEqual(in_bl), _, _, _)) + .WillOnce(Return(r)); + } + + void expect_get_snapcontext(const std::string& image_id, + const ::SnapContext &snapc, int r) { + bufferlist bl; + encode(snapc, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(librbd::util::header_name(image_id), _, StrEq("rbd"), + StrEq("get_snapcontext"), _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_snapshot_purge(MockSnapshotPurgeRequest &snapshot_purge_request, + const std::string &image_id, int r) { + EXPECT_CALL(snapshot_purge_request, construct(image_id)); + EXPECT_CALL(snapshot_purge_request, send()) + .WillOnce(Invoke([this, &snapshot_purge_request, r]() { + m_threads->work_queue->queue( + snapshot_purge_request.on_finish, r); + })); + } + + void expect_image_remove(MockLibrbdTrashRemoveRequest &image_remove_request, + const std::string &image_id, int r) { + EXPECT_CALL(image_remove_request, construct(image_id)); + EXPECT_CALL(image_remove_request, send()) + .WillOnce(Invoke([this, &image_remove_request, r]() { + m_threads->work_queue->queue( + image_remove_request.on_finish, r); + })); + } + + void expect_notify_image_removed(MockTrashWatcher& mock_trash_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_trash_watcher, notify_image_removed(image_id, _)) + .WillOnce(WithArg<1>(Invoke([this](Context *ctx) { + m_threads->work_queue->queue(ctx, 0); + }))); + } + +}; + +TEST_F(TestMockImageDeleterTrashRemoveRequest, Success) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", 0); + + MockLibrbdTrashRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_removed(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashDNE) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, -ENOENT); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, -EPERM); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashSourceIncorrect) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_USER, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashStateIncorrect) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + trash_image_spec.state = cls::rbd::TRASH_IMAGE_STATE_RESTORING; + expect_trash_get(trash_image_spec, 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EBUSY, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_RETRY_IMMEDIATELY, error_result); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashSetStateDNE) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", -ENOENT); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashSetStateError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", -EPERM); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, GetSnapContextDNE) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, -ENOENT); + + MockLibrbdTrashRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_removed(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, GetSnapContextError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, PurgeSnapshotBusy) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", -EBUSY); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EBUSY, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_RETRY_IMMEDIATELY, error_result); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, PurgeSnapshotError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, RemoveError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", 0); + + MockLibrbdTrashRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc new file mode 100644 index 000000000..a612dec14 --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc @@ -0,0 +1,519 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "librbd/TrashWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/TrashWatcher.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +struct MockTrashWatcher { + static MockTrashWatcher *s_instance; + static MockTrashWatcher &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + MockTrashWatcher() { + s_instance = this; + } + + MOCK_CONST_METHOD0(is_unregistered, bool()); + MOCK_METHOD1(register_watch, void(Context*)); + MOCK_METHOD1(unregister_watch, void(Context*)); +}; + +template <> +struct TrashWatcher<MockTestImageCtx> { + static TrashWatcher *s_instance; + + TrashWatcher(librados::IoCtx &io_ctx, ::MockContextWQ *work_queue) { + s_instance = this; + } + virtual ~TrashWatcher() { + } + + static TrashWatcher<MockTestImageCtx> &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + virtual void handle_rewatch_complete(int r) = 0; + + virtual void handle_image_added(const std::string &image_id, + const cls::rbd::TrashImageSpec& spec) = 0; + virtual void handle_image_removed(const std::string &image_id) = 0; + + bool is_unregistered() const { + return MockTrashWatcher::get_instance().is_unregistered(); + } + void register_watch(Context *ctx) { + MockTrashWatcher::get_instance().register_watch(ctx); + } + void unregister_watch(Context *ctx) { + MockTrashWatcher::get_instance().unregister_watch(ctx); + } +}; + +MockTrashWatcher *MockTrashWatcher::s_instance = nullptr; +TrashWatcher<MockTestImageCtx> *TrashWatcher<MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_deleter/TrashWatcher.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageDeleterTrashWatcher : public TestMockFixture { +public: + typedef TrashWatcher<librbd::MockTestImageCtx> MockTrashWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef librbd::MockTrashWatcher MockLibrbdTrashWatcher; + typedef librbd::TrashWatcher<librbd::MockTestImageCtx> LibrbdTrashWatcher; + + struct MockListener : TrashListener { + MOCK_METHOD2(handle_trash_image, void(const std::string&, + const ceph::real_clock::time_point&)); + }; + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_trash_watcher_is_unregistered(MockLibrbdTrashWatcher &mock_trash_watcher, + bool unregistered) { + EXPECT_CALL(mock_trash_watcher, is_unregistered()) + .WillOnce(Return(unregistered)); + } + + void expect_trash_watcher_register(MockLibrbdTrashWatcher &mock_trash_watcher, + int r) { + EXPECT_CALL(mock_trash_watcher, register_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_trash_watcher_unregister(MockLibrbdTrashWatcher &mock_trash_watcher, + int r) { + EXPECT_CALL(mock_trash_watcher, unregister_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_create_trash(librados::IoCtx &io_ctx, int r) { + EXPECT_CALL(get_mock_io_ctx(io_ctx), create(RBD_TRASH, false, _)) + .WillOnce(Return(r)); + } + + void expect_trash_list(librados::IoCtx &io_ctx, + const std::string& last_image_id, + std::map<std::string, cls::rbd::TrashImageSpec>&& images, + int r) { + bufferlist bl; + encode(last_image_id, bl); + encode(static_cast<size_t>(1024), bl); + + bufferlist out_bl; + encode(images, out_bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_TRASH, _, StrEq("rbd"), StrEq("trash_list"), + ContentsEqual(bl), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + + void expect_timer_add_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = + new LambdaContext([this, ctx](int r) { + std::lock_guard timer_locker{m_threads->timer_lock}; + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), + ReturnArg<1>())); + } + + void expect_handle_trash_image(MockListener& mock_listener, + const std::string& global_image_id) { + EXPECT_CALL(mock_listener, handle_trash_image(global_image_id, _)); + } + + int when_shut_down(MockTrashWatcher &mock_trash_watcher) { + C_SaferCond ctx; + mock_trash_watcher.shut_down(&ctx); + return ctx.wait(); + } + +}; + +TEST_F(TestMockImageDeleterTrashWatcher, EmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, NonEmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + MockListener mock_listener; + expect_handle_trash_image(mock_listener, "image0"); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + std::map<std::string, cls::rbd::TrashImageSpec> images; + images["image0"] = {cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "name", {}, {}}; + for (auto idx = 1; idx < 1024; ++idx) { + images["image" + stringify(idx)] = {}; + } + expect_trash_list(m_local_io_ctx, "", std::move(images), 0); + + images.clear(); + for (auto idx = 1024; idx < 2000; ++idx) { + images["image" + stringify(idx)] = {}; + } + expect_trash_list(m_local_io_ctx, "image999", std::move(images), 0); + + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, Notify) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + MockListener mock_listener; + expect_handle_trash_image(mock_listener, "image1"); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + LibrbdTrashWatcher::get_instance().handle_image_added( + "image1", {cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "name", {}, {}}); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, CreateBlocklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, -EBLOCKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLOCKLISTED, ctx.wait()); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, CreateDNE) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, -ENOENT); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, CreateError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, -EINVAL); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RegisterWatcherBlocklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, -EBLOCKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLOCKLISTED, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RegisterWatcherError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, -EINVAL); + expect_timer_add_event(mock_threads); + + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, TrashListBlocklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, -EBLOCKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLOCKLISTED, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, TrashListError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, -EINVAL); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, false); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, Rewatch) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, false); + expect_trash_list(m_local_io_ctx, "", {}, 0); + LibrbdTrashWatcher::get_instance().handle_rewatch_complete(0); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RewatchBlocklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + LibrbdTrashWatcher::get_instance().handle_rewatch_complete(-EBLOCKLISTED); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_map/test_Policy.cc b/src/test/rbd_mirror/image_map/test_Policy.cc new file mode 100644 index 000000000..5f8bdd17f --- /dev/null +++ b/src/test/rbd_mirror/image_map/test_Policy.cc @@ -0,0 +1,377 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/Context.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/image_map/Types.h" +#include "tools/rbd_mirror/image_map/SimplePolicy.h" +#include "include/stringify.h" +#include "common/Thread.h" + +void register_test_image_policy() { +} + +namespace rbd { +namespace mirror { +namespace image_map { + +class TestImageMapPolicy : public TestFixture { +public: + void SetUp() override { + TestFixture::SetUp(); + + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", + "0")); + + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + std::string policy_type = cct->_conf.get_val<string>("rbd_mirror_image_policy_type"); + + if (policy_type == "none" || policy_type == "simple") { + m_policy = image_map::SimplePolicy::create(m_local_io_ctx); + } else { + ceph_abort(); + } + + m_policy->init({}); + } + + void TearDown() override { + TestFixture::TearDown(); + delete m_policy; + } + + void map_image(const std::string &global_image_id) { + ASSERT_TRUE(m_policy->add_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + } + + void unmap_image(const std::string &global_image_id) { + ASSERT_TRUE(m_policy->remove_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_REMOVE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + } + + void shuffle_image(const std::string &global_image_id) { + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + } + + Policy *m_policy; +}; + +TEST_F(TestImageMapPolicy, NegativeLookup) { + const std::string global_image_id = "global id 1"; + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id == UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, Init) { + const std::string global_image_id = "global id 1"; + + m_policy->init({{global_image_id, {"9876", {}, {}}}}); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +TEST_F(TestImageMapPolicy, MapImage) { + const std::string global_image_id = "global id 1"; + + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, UnmapImage) { + const std::string global_image_id = "global id 1"; + + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + + // unmap image + unmap_image(global_image_id); + + info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id == UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, ShuffleImageAddInstance) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", "global id 6" + }; + + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + + for (auto const &global_image_id : shuffle_global_image_ids) { + shuffle_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } +} + +TEST_F(TestImageMapPolicy, ShuffleImageRemoveInstance) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5" + }; + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + + for (auto const &global_image_id : shuffle_global_image_ids) { + shuffle_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + // record which of the images got migrated to the new instance + std::set<std::string> remapped_global_image_ids; + for (auto const &global_image_id: shuffle_global_image_ids) { + LookupInfo info = m_policy->lookup(global_image_id); + if (info.instance_id == "9876") { + remapped_global_image_ids.emplace(global_image_id); + } + } + + shuffle_global_image_ids.clear(); + m_policy->remove_instances({"9876"}, &shuffle_global_image_ids); + + ASSERT_TRUE(shuffle_global_image_ids == remapped_global_image_ids); + + for (auto const &global_image_id : shuffle_global_image_ids) { + shuffle_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } +} + +TEST_F(TestImageMapPolicy, RetryMapUpdate) { + const std::string global_image_id = "global id 1"; + + ASSERT_TRUE(m_policy->add_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + // on-disk map update failed + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EIO)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, MapFailureAndUnmap) { + const std::string global_image_id = "global id 1"; + + ASSERT_TRUE(m_policy->add_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + m_policy->remove_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EBLOCKLISTED)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, -ENOENT)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_TRUE(m_policy->remove_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_REMOVE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +TEST_F(TestImageMapPolicy, ReshuffleWithMapFailure) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6" + }; + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_FALSE(shuffle_global_image_ids.empty()); + + const std::string global_image_id = *(shuffle_global_image_ids.begin()); + shuffle_global_image_ids.clear(); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + // peer unavailable + m_policy->remove_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EBLOCKLISTED)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +TEST_F(TestImageMapPolicy, ShuffleFailureAndRemove) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6" + }; + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_FALSE(shuffle_global_image_ids.empty()); + + std::string global_image_id = *(shuffle_global_image_ids.begin()); + shuffle_global_image_ids.clear(); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + // peer unavailable + m_policy->remove_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EBLOCKLISTED)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_TRUE(m_policy->remove_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_REMOVE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id == UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, InitialInstanceUpdate) { + const std::string global_image_id = "global id 1"; + + m_policy->init({{global_image_id, {"9876", {}, {}}}}); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + auto instance_id = stringify(m_local_io_ctx.get_instance_id()); + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({instance_id}, &shuffle_global_image_ids); + + ASSERT_EQ(0U, shuffle_global_image_ids.size()); + ASSERT_TRUE(m_policy->finish_action(global_image_id, -ENOENT)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +} // namespace image_map +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc new file mode 100644 index 000000000..cc2267160 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_CreateLocalImageRequest.cc @@ -0,0 +1,341 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include <boost/intrusive_ptr.hpp> + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal + +namespace util { + +static std::string s_image_id; + +template <> +std::string generate_image_id<MockTestImageCtx>(librados::IoCtx&) { + ceph_assert(!s_image_id.empty()); + return s_image_id; +} + +} // namespace util +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { +}; + +namespace image_replayer { + +template<> +struct CreateImageRequest<librbd::MockTestImageCtx> { + static CreateImageRequest* s_instance; + Context *on_finish = nullptr; + + static CreateImageRequest* create(Threads<librbd::MockTestImageCtx>* threads, + librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + librbd::MockTestImageCtx *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + s_instance->construct(local_image_id); + return s_instance; + } + + CreateImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~CreateImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); +}; + +CreateImageRequest<librbd::MockTestImageCtx>* + CreateImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +namespace journal { + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + std::string local_image_id; + + std::string remote_mirror_uuid; + ::journal::MockJournalerProxy* remote_journaler = nullptr; + cls::journal::ClientState remote_client_state; + librbd::journal::MirrorPeerClientMeta remote_client_meta; +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc" + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::WithArg; + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +class TestMockImageReplayerJournalCreateLocalImageRequest : public TestMockFixture { +public: + typedef CreateLocalImageRequest<librbd::MockTestImageCtx> MockCreateLocalImageRequest; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef CreateImageRequest<librbd::MockTestImageCtx> MockCreateImageRequest; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + m_mock_remote_image_ctx = new librbd::MockTestImageCtx(*m_remote_image_ctx); + } + + void TearDown() override { + delete m_mock_remote_image_ctx; + TestMockFixture::TearDown(); + } + + void expect_journaler_register_client( + ::journal::MockJournaler& mock_journaler, + const librbd::journal::ClientData& client_data, int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_journaler_unregister_client( + ::journal::MockJournaler& mock_journaler, int r) { + EXPECT_CALL(mock_journaler, unregister_client(_)) + .WillOnce(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })); + } + + void expect_journaler_update_client( + ::journal::MockJournaler& mock_journaler, + const librbd::journal::ClientData& client_data, int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, update_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_create_image(MockCreateImageRequest& mock_create_image_request, + const std::string& image_id, int r) { + EXPECT_CALL(mock_create_image_request, construct(image_id)); + EXPECT_CALL(mock_create_image_request, send()) + .WillOnce(Invoke([this, &mock_create_image_request, r]() { + m_threads->work_queue->queue(mock_create_image_request.on_finish, r); + })); + } + + MockCreateLocalImageRequest* create_request( + MockThreads& mock_threads, + MockStateBuilder& mock_state_builder, + const std::string& global_image_id, + Context* on_finish) { + return new MockCreateLocalImageRequest( + &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx, + global_image_id, nullptr, nullptr, &mock_state_builder, + on_finish); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::MockTestImageCtx *m_mock_remote_image_ctx = nullptr; +}; + +TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, Success) { + InSequence seq; + + // re-register the client + ::journal::MockJournaler mock_journaler; + expect_journaler_unregister_client(mock_journaler, 0); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + librbd::util::s_image_id = "local image id"; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data; + client_data.client_meta = mirror_peer_client_meta; + expect_journaler_register_client(mock_journaler, client_data, 0); + + // create the missing local image + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, "local image id", 0); + + C_SaferCond ctx; + MockThreads mock_threads; + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ("local image id", mock_state_builder.local_image_id); + ASSERT_EQ("local image id", mock_state_builder.remote_client_meta.image_id); + ASSERT_EQ(librbd::journal::MIRROR_PEER_STATE_SYNCING, + mock_state_builder.remote_client_meta.state); +} + +TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, UnregisterError) { + InSequence seq; + + // re-register the client + ::journal::MockJournaler mock_journaler; + expect_journaler_unregister_client(mock_journaler, -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads; + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, RegisterError) { + InSequence seq; + + // re-register the client + ::journal::MockJournaler mock_journaler; + expect_journaler_unregister_client(mock_journaler, 0); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + librbd::util::s_image_id = "local image id"; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data; + client_data.client_meta = mirror_peer_client_meta; + expect_journaler_register_client(mock_journaler, client_data, -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads; + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, CreateImageError) { + InSequence seq; + + // re-register the client + ::journal::MockJournaler mock_journaler; + expect_journaler_unregister_client(mock_journaler, 0); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + librbd::util::s_image_id = "local image id"; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data; + client_data.client_meta = mirror_peer_client_meta; + expect_journaler_register_client(mock_journaler, client_data, 0); + + // create the missing local image + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, "local image id", -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads; + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalCreateLocalImageRequest, CreateImageDuplicate) { + InSequence seq; + + // re-register the client + ::journal::MockJournaler mock_journaler; + expect_journaler_unregister_client(mock_journaler, 0); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + librbd::util::s_image_id = "local image id"; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data; + client_data.client_meta = mirror_peer_client_meta; + expect_journaler_register_client(mock_journaler, client_data, 0); + + // create the missing local image + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, "local image id", -EBADF); + + // re-register the client + expect_journaler_unregister_client(mock_journaler, 0); + expect_journaler_register_client(mock_journaler, client_data, 0); + + // re-create the local image + expect_create_image(mock_create_image_request, "local image id", 0); + + C_SaferCond ctx; + MockThreads mock_threads; + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_EventPreprocessor.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_EventPreprocessor.cc new file mode 100644 index 000000000..ad0055281 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_EventPreprocessor.cc @@ -0,0 +1,266 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/journal/EventPreprocessor.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/journal/EventPreprocessor.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using testing::_; +using testing::WithArg; + +class TestMockImageReplayerJournalEventPreprocessor : public TestMockFixture { +public: + typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_image_refresh(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.state, refresh(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_update_client(::journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, update_client(_, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + + librbd::ImageCtx *m_local_image_ctx; + librbd::journal::MirrorPeerClientMeta m_client_meta; + +}; + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, IsNotRequired) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + ASSERT_FALSE(event_preprocessor.is_required(event_entry)); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, IsRequiredSnapMapPrune) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + m_client_meta.snap_seqs = {{1, 2}, {3, 4}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + ASSERT_TRUE(event_preprocessor.is_required(event_entry)); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, IsRequiredSnapRename) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::SnapRenameEvent{}}; + ASSERT_TRUE(event_preprocessor.is_required(event_entry)); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, PreprocessSnapMapPrune) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + expect_update_client(mock_remote_journaler, 0); + + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + m_client_meta.snap_seqs = {{1, 2}, {3, 4}, {5, 6}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(0, ctx.wait()); + + librbd::SnapSeqs expected_snap_seqs = {{5, 6}}; + ASSERT_EQ(expected_snap_seqs, m_client_meta.snap_seqs); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, PreprocessSnapRename) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + expect_update_client(mock_remote_journaler, 0); + + mock_local_image_ctx.snap_ids = {{{cls::rbd::UserSnapshotNamespace(), "snap"}, 6}}; + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(0, ctx.wait()); + + librbd::SnapSeqs expected_snap_seqs = {{5, 6}}; + ASSERT_EQ(expected_snap_seqs, m_client_meta.snap_seqs); + + librbd::journal::SnapRenameEvent *event = + boost::get<librbd::journal::SnapRenameEvent>(&event_entry.event); + ASSERT_EQ(6U, event->snap_id); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, PreprocessSnapRenameMissing) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + librbd::journal::SnapRenameEvent *event = + boost::get<librbd::journal::SnapRenameEvent>(&event_entry.event); + ASSERT_EQ(CEPH_NOSNAP, event->snap_id); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, PreprocessSnapRenameKnown) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + m_client_meta.snap_seqs = {{5, 6}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(0, ctx.wait()); + + librbd::SnapSeqs expected_snap_seqs = {{5, 6}}; + ASSERT_EQ(expected_snap_seqs, m_client_meta.snap_seqs); + + librbd::journal::SnapRenameEvent *event = + boost::get<librbd::journal::SnapRenameEvent>(&event_entry.event); + ASSERT_EQ(6U, event->snap_id); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, PreprocessRefreshError) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, -EINVAL); + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalEventPreprocessor, PreprocessClientUpdateError) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + expect_update_client(mock_remote_journaler, -EINVAL); + + mock_local_image_ctx.snap_ids = {{{cls::rbd::UserSnapshotNamespace(), "snap"}, 6}}; + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_PrepareReplayRequest.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_PrepareReplayRequest.cc new file mode 100644 index 000000000..4aa951629 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_PrepareReplayRequest.cc @@ -0,0 +1,751 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal +} // namespace rbd + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + StateBuilder(librbd::MockTestImageCtx& local_image_ctx, + ::journal::MockJournaler& remote_journaler, + const librbd::journal::MirrorPeerClientMeta& remote_client_meta) + : local_image_ctx(&local_image_ctx), + local_image_id(local_image_ctx.id), + remote_journaler(&remote_journaler), + remote_client_meta(remote_client_meta) { + } + + librbd::MockTestImageCtx* local_image_ctx; + std::string local_image_id; + + std::string remote_mirror_uuid = "remote mirror uuid"; + ::journal::MockJournaler* remote_journaler = nullptr; + librbd::journal::MirrorPeerClientMeta remote_client_meta; +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageReplayerJournalPrepareReplayRequest : public TestMockFixture { +public: + typedef PrepareReplayRequest<librbd::MockTestImageCtx> MockPrepareReplayRequest; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef std::list<cls::journal::Tag> Tags; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_update_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, update_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_journaler_get_tags(::journal::MockJournaler &mock_journaler, + uint64_t tag_class, const Tags& tags, + int r) { + EXPECT_CALL(mock_journaler, get_tags(tag_class, _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([tags](Tags *out_tags) { + *out_tags = tags; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journal_get_tag_tid(librbd::MockJournal &mock_journal, + uint64_t tag_tid) { + EXPECT_CALL(mock_journal, get_tag_tid()).WillOnce(Return(tag_tid)); + } + + void expect_journal_get_tag_data(librbd::MockJournal &mock_journal, + const librbd::journal::TagData &tag_data) { + EXPECT_CALL(mock_journal, get_tag_data()).WillOnce(Return(tag_data)); + } + + void expect_is_resync_requested(librbd::MockJournal &mock_journal, + bool do_resync, int r) { + EXPECT_CALL(mock_journal, is_resync_requested(_)) + .WillOnce(DoAll(SetArgPointee<0>(do_resync), + Return(r))); + } + + bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { + bufferlist bl; + encode(tag_data, bl); + return bl; + } + + MockPrepareReplayRequest* create_request( + MockStateBuilder& mock_state_builder, + const std::string& local_mirror_uuid, + bool* resync_requested, bool* syncing, Context* on_finish) { + return new MockPrepareReplayRequest( + local_mirror_uuid, nullptr, &mock_state_builder, resync_requested, + syncing, on_finish); + } + + librbd::ImageCtx *m_local_image_ctx = nullptr; +}; + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, Success) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // single promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 344, 99})}, + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, NoLocalJournal) { + InSequence seq; + + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, ResyncRequested) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, true, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, ResyncRequestedError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, -EINVAL); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, Syncing) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + C_SaferCond ctx; + ::journal::MockJournaler mock_remote_journaler; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_TRUE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, GetRemoteTagClassError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, -EINVAL); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, GetRemoteTagsError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // single promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 344, 99})}, + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, -EINVAL); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, LocalDemotedRemoteSyncingState) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::ORPHAN_MIRROR_UUID, + "remote mirror uuid", true, 4, 1}); + + // update client state + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data; + client_data.client_meta = mirror_peer_client_meta; + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_update_client(mock_remote_journaler, client_data, 0); + + // lookup remote image tag class + client_data = {librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 4, 1})}, + {6, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 5, 1})}, + {7, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 6, 1})}, + {8, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 7, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, UpdateClientError) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // single promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 344, 99})}, + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, RemoteDemotePromote) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 2, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 4, 369})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, MultipleRemoteDemotePromotes) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::ORPHAN_MIRROR_UUID, + "remote mirror uuid", true, 4, 1}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 4, 1})}, + {6, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 5, 1})}, + {7, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 6, 1})}, + {8, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 7, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, LocalDemoteRemotePromote) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 346); + expect_journal_get_tag_data(mock_journal, + {librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 345, 1}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({"local mirror uuid", "local mirror uuid", + true, 344, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + "local mirror uuid", true, 345, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_FALSE(resync_requested); + ASSERT_FALSE(syncing); +} + +TEST_F(TestMockImageReplayerJournalPrepareReplayRequest, SplitBrainForcePromote) { + InSequence seq; + + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + // check initial state + expect_is_resync_requested(mock_journal, false, 0); + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 344, 0}); + + // lookup remote image tag class + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + cls::journal::Client client; + encode(client_data, client.data); + ::journal::MockJournaler mock_remote_journaler; + expect_journaler_get_client(mock_remote_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})} + }; + expect_journaler_get_tags(mock_remote_journaler, 123, tags, 0); + + C_SaferCond ctx; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + mirror_peer_client_meta); + bool resync_requested; + bool syncing; + auto request = create_request(mock_state_builder, "local mirror uuid", + &resync_requested, &syncing, &ctx); + request->send(); + ASSERT_EQ(-EEXIST, ctx.wait()); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc new file mode 100644 index 000000000..211216eeb --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/journal/test_mock_Replayer.cc @@ -0,0 +1,2160 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/ReplayerListener.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_replayer/journal/Replayer.h" +#include "tools/rbd_mirror/image_replayer/journal/EventPreprocessor.h" +#include "tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include <boost/intrusive_ptr.hpp> + +namespace librbd { + +namespace { + +struct MockTestJournal; + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx, + MockTestJournal& mock_test_journal) + : librbd::MockImageCtx(image_ctx), journal(&mock_test_journal) { + } + + MockTestJournal* journal = nullptr; +}; + +struct MockTestJournal : public MockJournal { + MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **, + Context *on_start)); + MOCK_METHOD0(stop_external_replay, void()); +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; + typedef ::journal::MockReplayEntryProxy ReplayEntry; +}; + +template<> +struct Replay<MockTestImageCtx> { + MOCK_METHOD2(decode, int(bufferlist::const_iterator *, EventEntry *)); + MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *)); + MOCK_METHOD1(flush, void(Context*)); + MOCK_METHOD2(shut_down, void(bool, Context*)); +}; + +} // namespace journal +} // namespace librbd + +namespace boost { + +template<> +struct intrusive_ptr<librbd::MockTestJournal> { + intrusive_ptr() { + } + intrusive_ptr(librbd::MockTestJournal* mock_test_journal) + : mock_test_journal(mock_test_journal) { + } + + librbd::MockTestJournal* operator->() { + return mock_test_journal; + } + + void reset() { + mock_test_journal = nullptr; + } + + const librbd::MockTestJournal* get() const { + return mock_test_journal; + } + + template<typename T> + bool operator==(T* t) const { + return (mock_test_journal == t); + } + + librbd::MockTestJournal* mock_test_journal = nullptr; +}; + +} // namespace boost + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx>* threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +namespace { + +struct MockReplayerListener : public image_replayer::ReplayerListener { + MOCK_METHOD0(handle_notification, void()); +}; + +} // anonymous namespace + +namespace image_replayer { + +template<> +struct CloseImageRequest<librbd::MockTestImageCtx> { + static CloseImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + return s_instance; + } + + CloseImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~CloseImageRequest() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +namespace journal { + +template <> +struct EventPreprocessor<librbd::MockTestImageCtx> { + static EventPreprocessor *s_instance; + + static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx, + ::journal::MockJournaler &remote_journaler, + const std::string &local_mirror_uuid, + librbd::journal::MirrorPeerClientMeta *client_meta, + MockContextWQ *work_queue) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + static void destroy(EventPreprocessor* processor) { + } + + EventPreprocessor() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~EventPreprocessor() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &)); + MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *)); +}; + +template<> +struct ReplayStatusFormatter<librbd::MockTestImageCtx> { + static ReplayStatusFormatter* s_instance; + + static ReplayStatusFormatter* create(::journal::MockJournaler *journaler, + const std::string &mirror_uuid) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + static void destroy(ReplayStatusFormatter* formatter) { + } + + ReplayStatusFormatter() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~ReplayStatusFormatter() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD1(handle_entry_processed, void(uint64_t)); + MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish)); +}; + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + StateBuilder(librbd::MockTestImageCtx& local_image_ctx, + ::journal::MockJournaler& remote_journaler, + const librbd::journal::MirrorPeerClientMeta& remote_client_meta) + : local_image_ctx(&local_image_ctx), + remote_journaler(&remote_journaler), + remote_client_meta(remote_client_meta) { + } + + librbd::MockTestImageCtx* local_image_ctx; + std::string remote_mirror_uuid = "remote mirror uuid"; + ::journal::MockJournaler* remote_journaler = nullptr; + librbd::journal::MirrorPeerClientMeta remote_client_meta; +}; + +EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr; +ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_replayer/journal/Replayer.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::MatcherCast; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::SaveArg; +using ::testing::SetArgPointee; +using ::testing::WithArg; + +class TestMockImageReplayerJournalReplayer : public TestMockFixture { +public: + typedef Replayer<librbd::MockTestImageCtx> MockReplayer; + typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor; + typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest; + typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { + bufferlist bl; + encode(tag_data, bl); + return bl; + } + + void expect_work_queue_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event_after_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillRepeatedly( + DoAll(Invoke([this](double seconds, Context *ctx) { + m_threads->timer->add_event_after(seconds, ctx); + }), + ReturnArg<1>())); + EXPECT_CALL(*mock_threads.timer, cancel_event(_)) + .WillRepeatedly( + Invoke([this](Context *ctx) { + return m_threads->timer->cancel_event(ctx); + })); + } + + void expect_init(::journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, init(_)) + .WillOnce(CompleteContext(m_threads->work_queue, r)); + } + + void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, stop_replay(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) { + EXPECT_CALL(mock_replay, shut_down(cancel_ops, _)) + .WillOnce(WithArg<1>(CompleteContext(m_threads->work_queue, r))); + } + + void expect_get_cached_client(::journal::MockJournaler &mock_journaler, + const std::string& client_id, + const cls::journal::Client& client, + const librbd::journal::ClientMeta& client_meta, + int r) { + librbd::journal::ClientData client_data; + client_data.client_meta = client_meta; + + cls::journal::Client client_copy{client}; + encode(client_data, client_copy.data); + + EXPECT_CALL(mock_journaler, get_cached_client(client_id, _)) + .WillOnce(DoAll(SetArgPointee<1>(client_copy), + Return(r))); + } + + void expect_start_external_replay(librbd::MockTestJournal &mock_journal, + MockReplay *mock_replay, int r) { + EXPECT_CALL(mock_journal, start_external_replay(_, _)) + .WillOnce(DoAll(SetArgPointee<0>(mock_replay), + WithArg<1>(CompleteContext(m_threads->work_queue, r)))); + } + + void expect_is_tag_owner(librbd::MockTestJournal &mock_journal, + bool is_owner) { + EXPECT_CALL(mock_journal, is_tag_owner()).WillOnce(Return(is_owner)); + } + + void expect_is_resync_requested(librbd::MockTestJournal &mock_journal, + int r, bool resync_requested) { + EXPECT_CALL(mock_journal, is_resync_requested(_)).WillOnce( + DoAll(SetArgPointee<0>(resync_requested), + Return(r))); + } + + void expect_get_commit_tid_in_debug( + ::journal::MockReplayEntry &mock_replay_entry) { + // It is used in debug messages and depends on debug level + EXPECT_CALL(mock_replay_entry, get_commit_tid()) + .Times(AtLeast(0)) + .WillRepeatedly(Return(0)); + } + + void expect_get_tag_tid_in_debug(librbd::MockTestJournal &mock_journal) { + // It is used in debug messages and depends on debug level + EXPECT_CALL(mock_journal, get_tag_tid()).Times(AtLeast(0)) + .WillRepeatedly(Return(0)); + } + + void expect_committed(::journal::MockReplayEntry &mock_replay_entry, + ::journal::MockJournaler &mock_journaler, int times) { + EXPECT_CALL(mock_replay_entry, get_data()).Times(times); + EXPECT_CALL(mock_journaler, committed( + MatcherCast<const ::journal::MockReplayEntryProxy&>(_))) + .Times(times); + } + + void expect_try_pop_front(::journal::MockJournaler &mock_journaler, + uint64_t replay_tag_tid, bool entries_available) { + EXPECT_CALL(mock_journaler, try_pop_front(_, _)) + .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()), + SetArgPointee<1>(replay_tag_tid), + Return(entries_available))); + } + + void expect_try_pop_front_return_no_entries( + ::journal::MockJournaler &mock_journaler, Context *on_finish) { + EXPECT_CALL(mock_journaler, try_pop_front(_, _)) + .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e, + uint64_t *t) { + on_finish->complete(0); + }), + Return(false))); + } + + void expect_get_tag(::journal::MockJournaler &mock_journaler, + const cls::journal::Tag &tag, int r) { + EXPECT_CALL(mock_journaler, get_tag(_, _, _)) + .WillOnce(DoAll(SetArgPointee<1>(tag), + WithArg<2>(CompleteContext(r)))); + } + + void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) { + EXPECT_CALL(mock_journal, allocate_tag(_, _, _)) + .WillOnce(WithArg<2>(CompleteContext(r))); + } + + void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor, + bool required, int r) { + EXPECT_CALL(mock_event_preprocessor, is_required(_)) + .WillOnce(Return(required)); + if (required) { + EXPECT_CALL(mock_event_preprocessor, preprocess(_, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + } + + void expect_process(MockReplay &mock_replay, + int on_ready_r, int on_commit_r) { + EXPECT_CALL(mock_replay, process(_, _, _)) + .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)), + WithArg<2>(CompleteContext(on_commit_r)))); + } + + void expect_flush(MockReplay& mock_replay, int r) { + EXPECT_CALL(mock_replay, flush(_)) + .WillOnce(CompleteContext(m_threads->work_queue, r)); + } + + void expect_flush_commit_position(::journal::MockJournaler& mock_journal, + int r) { + EXPECT_CALL(mock_journal, flush_commit_position(_)) + .WillOnce(CompleteContext(m_threads->work_queue, r)); + } + + void expect_get_tag_data(librbd::MockTestJournal& mock_local_journal, + const librbd::journal::TagData& tag_data) { + EXPECT_CALL(mock_local_journal, get_tag_data()) + .WillOnce(Return(tag_data)); + } + + void expect_send(MockCloseImageRequest &mock_close_image_request, int r) { + EXPECT_CALL(mock_close_image_request, send()) + .WillOnce(Invoke([this, &mock_close_image_request, r]() { + *mock_close_image_request.image_ctx = nullptr; + m_threads->work_queue->queue(mock_close_image_request.on_finish, r); + })); + } + + void expect_notification(MockThreads& mock_threads, + MockReplayerListener& mock_replayer_listener) { + EXPECT_CALL(mock_replayer_listener, handle_notification()) + .WillOnce(Invoke([this]() { + std::unique_lock locker{m_lock}; + m_notified = true; + m_cond.notify_all(); + })); + } + + int wait_for_notification() { + std::unique_lock locker{m_lock}; + while (!m_notified) { + if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) { + return -ETIMEDOUT; + } + } + m_notified = false; + return 0; + } + + void expect_local_journal_add_listener( + librbd::MockTestJournal& mock_local_journal, + librbd::journal::Listener** local_journal_listener) { + EXPECT_CALL(mock_local_journal, add_listener(_)) + .WillOnce(SaveArg<0>(local_journal_listener)); + expect_is_tag_owner(mock_local_journal, false); + expect_is_resync_requested(mock_local_journal, 0, false); + } + + int init_entry_replayer(MockReplayer& mock_replayer, + MockThreads& mock_threads, + MockReplayerListener& mock_replayer_listener, + librbd::MockTestJournal& mock_local_journal, + ::journal::MockJournaler& mock_remote_journaler, + MockReplay& mock_local_journal_replay, + librbd::journal::Listener** local_journal_listener, + ::journal::ReplayHandler** remote_replay_handler, + ::journal::JournalMetadataListener** remote_journal_listener) { + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)) + .WillOnce(SaveArg<0>(remote_journal_listener)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", {}, + {librbd::journal::MirrorPeerClientMeta{}}, 0); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + local_journal_listener); + EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _)) + .WillOnce(SaveArg<0>(remote_replay_handler)); + expect_notification(mock_threads, mock_replayer_listener); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + int r = init_ctx.wait(); + if (r < 0) { + return r; + } + + return wait_for_notification(); + } + + int shut_down_entry_replayer(MockReplayer& mock_replayer, + MockThreads& mock_threads, + librbd::MockTestJournal& mock_local_journal, + ::journal::MockJournaler& mock_remote_journaler, + MockReplay& mock_local_journal_replay) { + expect_shut_down(mock_local_journal_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + return shutdown_ctx.wait(); + } + + librbd::ImageCtx* m_local_image_ctx = nullptr; + + ceph::mutex m_lock = ceph::make_mutex( + "TestMockImageReplayerJournalReplayer"); + ceph::condition_variable m_cond; + bool m_notified = false; +}; + +TEST_F(TestMockImageReplayerJournalReplayer, InitShutDown) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitRemoteJournalerError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, -EINVAL); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-EINVAL, init_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitRemoteJournalerGetClientError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", {}, + {librbd::journal::MirrorPeerClientMeta{}}, -EINVAL); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-EINVAL, init_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitNoLocalJournal) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + + mock_local_image_ctx.journal = nullptr; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", {}, + {librbd::journal::MirrorPeerClientMeta{}}, 0); + + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-EINVAL, init_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitLocalJournalStartExternalReplayError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", {}, + {librbd::journal::MirrorPeerClientMeta{}}, 0); + expect_start_external_replay(mock_local_journal, nullptr, -EINVAL); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-EINVAL, init_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitIsPromoted) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", {}, + {librbd::journal::MirrorPeerClientMeta{}}, 0); + MockReplay mock_local_journal_replay; + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + EXPECT_CALL(mock_local_journal, add_listener(_)); + expect_is_tag_owner(mock_local_journal, true); + expect_notification(mock_threads, mock_replayer_listener); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + ASSERT_EQ(0, wait_for_notification()); + + expect_shut_down(mock_local_journal_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitDisconnected) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + mock_local_image_ctx.config.set_val("rbd_mirroring_resync_after_disconnect", + "false"); + + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", + {{}, {}, {}, + cls::journal::CLIENT_STATE_DISCONNECTED}, + {librbd::journal::MirrorPeerClientMeta{ + mock_local_image_ctx.id}}, 0); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-ENOTCONN, init_ctx.wait()); + ASSERT_FALSE(mock_replayer.is_resync_requested()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitDisconnectedResync) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + mock_local_image_ctx.config.set_val("rbd_mirroring_resync_after_disconnect", + "true"); + + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", + {{}, {}, {}, + cls::journal::CLIENT_STATE_DISCONNECTED}, + {librbd::journal::MirrorPeerClientMeta{ + mock_local_image_ctx.id}}, 0); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-ENOTCONN, init_ctx.wait()); + ASSERT_TRUE(mock_replayer.is_resync_requested()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitResyncRequested) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", {}, + {librbd::journal::MirrorPeerClientMeta{}}, 0); + MockReplay mock_local_journal_replay; + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + EXPECT_CALL(mock_local_journal, add_listener(_)); + expect_is_tag_owner(mock_local_journal, false); + expect_is_resync_requested(mock_local_journal, 0, true); + expect_notification(mock_threads, mock_replayer_listener); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + ASSERT_EQ(0, wait_for_notification()); + + expect_shut_down(mock_local_journal_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, InitResyncRequestedError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + expect_init(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", {}, + {librbd::journal::MirrorPeerClientMeta{}}, 0); + MockReplay mock_local_journal_replay; + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + EXPECT_CALL(mock_local_journal, add_listener(_)); + expect_is_tag_owner(mock_local_journal, false); + expect_is_resync_requested(mock_local_journal, -EINVAL, false); + expect_notification(mock_threads, mock_replayer_listener); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + ASSERT_EQ(0, wait_for_notification()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + expect_shut_down(mock_local_journal_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, ShutDownLocalJournalReplayError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_shut_down(mock_local_journal_replay, true, -EINVAL); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(-EINVAL, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, CloseLocalImageError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_shut_down(mock_local_journal_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, -EINVAL); + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(-EINVAL, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, StopRemoteJournalerError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_shut_down(mock_local_journal_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + expect_stop_replay(mock_remote_journaler, -EPERM); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(-EPERM, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, Replay) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + expect_committed(mock_replay_entry, mock_remote_journaler, 2); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + + // replay_flush + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + + // process + EXPECT_CALL(mock_local_journal_replay, decode(_, _)).WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, false, 0); + expect_process(mock_local_journal_replay, 0, 0); + EXPECT_CALL(mock_replay_status_formatter, handle_entry_processed(_)); + + // the next event with preprocess + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + EXPECT_CALL(mock_local_journal_replay, decode(_, _)).WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, true, 0); + expect_process(mock_local_journal_replay, 0, 0); + EXPECT_CALL(mock_replay_status_formatter, handle_entry_processed(_)); + + // attempt to process the next event + C_SaferCond replay_ctx; + expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx); + + // fire + remote_replay_handler->handle_entries_available(); + ASSERT_EQ(0, replay_ctx.wait()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, DecodeError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + + // replay_flush + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + + // process + EXPECT_CALL(mock_replay_entry, get_data()); + EXPECT_CALL(mock_local_journal_replay, decode(_, _)) + .WillOnce(Return(-EINVAL)); + expect_notification(mock_threads, mock_replayer_listener); + + // fire + remote_replay_handler->handle_entries_available(); + wait_for_notification(); + + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, DelayedReplay) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + expect_committed(mock_replay_entry, mock_remote_journaler, 1); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + + // replay_flush + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + + // process with delay + EXPECT_CALL(mock_replay_entry, get_data()); + librbd::journal::EventEntry event_entry( + librbd::journal::AioDiscardEvent(123, 345, 0), ceph_clock_now()); + EXPECT_CALL(mock_local_journal_replay, decode(_, _)) + .WillOnce(DoAll(SetArgPointee<1>(event_entry), + Return(0))); + + Context* delayed_task_ctx = nullptr; + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce( + DoAll(Invoke([this, &delayed_task_ctx](double seconds, Context *ctx) { + std::unique_lock locker{m_lock}; + delayed_task_ctx = ctx; + m_cond.notify_all(); + }), + ReturnArg<1>())); + expect_preprocess(mock_event_preprocessor, false, 0); + expect_process(mock_local_journal_replay, 0, 0); + EXPECT_CALL(mock_replay_status_formatter, handle_entry_processed(_)); + + // attempt to process the next event + C_SaferCond replay_ctx; + expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx); + + // fire + mock_local_image_ctx.mirroring_replay_delay = 600; + remote_replay_handler->handle_entries_available(); + { + std::unique_lock locker{m_lock}; + while (delayed_task_ctx == nullptr) { + if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) { + FAIL() << "timed out waiting for task"; + break; + } + } + } + { + std::unique_lock timer_locker{mock_threads.timer_lock}; + delayed_task_ctx->complete(0); + } + ASSERT_EQ(0, replay_ctx.wait()); + + // add a pending (delayed) entry before stop + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + C_SaferCond decode_ctx; + EXPECT_CALL(mock_local_journal_replay, decode(_, _)) + .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::const_iterator* it, + librbd::journal::EventEntry *e) { + decode_ctx.complete(0); + }), + Return(0))); + + remote_replay_handler->handle_entries_available(); + ASSERT_EQ(0, decode_ctx.wait()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, ReplayNoMemoryError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_notification(mock_threads, mock_replayer_listener); + remote_replay_handler->handle_complete(-ENOMEM); + + wait_for_notification(); + ASSERT_EQ(false, mock_replayer.is_replaying()); + ASSERT_EQ(-ENOMEM, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, LocalJournalForcePromoted) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_notification(mock_threads, mock_replayer_listener); + local_journal_listener->handle_promoted(); + wait_for_notification(); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, LocalJournalResyncRequested) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_notification(mock_threads, mock_replayer_listener); + local_journal_listener->handle_resync(); + wait_for_notification(); + + ASSERT_TRUE(mock_replayer.is_resync_requested()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, RemoteJournalDisconnected) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + mock_local_image_ctx.config.set_val("rbd_mirroring_resync_after_disconnect", + "true"); + + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_get_cached_client(mock_remote_journaler, "local mirror uuid", + {{}, {}, {}, + cls::journal::CLIENT_STATE_DISCONNECTED}, + {librbd::journal::MirrorPeerClientMeta{ + mock_local_image_ctx.id}}, 0); + expect_notification(mock_threads, mock_replayer_listener); + + remote_journaler_listener->handle_update(nullptr); + wait_for_notification(); + + ASSERT_EQ(-ENOTCONN, mock_replayer.get_error_code()); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_TRUE(mock_replayer.is_resync_requested()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, Flush) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_flush(mock_local_journal_replay, 0); + expect_flush_commit_position(mock_remote_journaler, 0); + + C_SaferCond ctx; + mock_replayer.flush(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, FlushError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_flush(mock_local_journal_replay, -EINVAL); + + C_SaferCond ctx; + mock_replayer.flush(&ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, FlushCommitPositionError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_flush(mock_local_journal_replay, 0); + expect_flush_commit_position(mock_remote_journaler, -EINVAL); + + C_SaferCond ctx; + mock_replayer.flush(&ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + + +TEST_F(TestMockImageReplayerJournalReplayer, ReplayFlushShutDownError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_try_pop_front(mock_remote_journaler, 1, true); + expect_shut_down(mock_local_journal_replay, false, -EINVAL); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_notification(mock_threads, mock_replayer_listener); + remote_replay_handler->handle_entries_available(); + + wait_for_notification(); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, ReplayFlushStartError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + expect_try_pop_front(mock_remote_journaler, 1, true); + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, nullptr, -EINVAL); + expect_notification(mock_threads, mock_replayer_listener); + remote_replay_handler->handle_entries_available(); + + wait_for_notification(); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + MockCloseImageRequest mock_close_image_request; + expect_send(mock_close_image_request, 0); + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerJournalReplayer, GetTagError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, -EINVAL); + expect_notification(mock_threads, mock_replayer_listener); + remote_replay_handler->handle_entries_available(); + + wait_for_notification(); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, AllocateTagDemotion) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_notification(mock_threads, mock_replayer_listener); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + expect_committed(mock_replay_entry, mock_remote_journaler, 1); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_get_tag_data(mock_local_journal, {}); + expect_allocate_tag(mock_local_journal, 0); + EXPECT_CALL(mock_local_journal_replay, decode(_, _)).WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, false, 0); + expect_process(mock_local_journal_replay, 0, 0); + EXPECT_CALL(mock_replay_status_formatter, handle_entry_processed(_)); + + remote_replay_handler->handle_entries_available(); + wait_for_notification(); + ASSERT_FALSE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, AllocateTagError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, -EINVAL); + expect_notification(mock_threads, mock_replayer_listener); + remote_replay_handler->handle_entries_available(); + + wait_for_notification(); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, PreprocessError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + EXPECT_CALL(mock_replay_entry, get_data()); + EXPECT_CALL(mock_local_journal_replay, decode(_, _)).WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, true, -EINVAL); + + expect_notification(mock_threads, mock_replayer_listener); + remote_replay_handler->handle_entries_available(); + + wait_for_notification(); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, ProcessError) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + EXPECT_CALL(mock_replay_entry, get_data()); + EXPECT_CALL(mock_local_journal_replay, decode(_, _)).WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, false, 0); + expect_process(mock_local_journal_replay, 0, -EINVAL); + EXPECT_CALL(mock_replay_status_formatter, handle_entry_processed(_)); + + // attempt to process the next event + C_SaferCond replay_ctx; + expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx); + remote_replay_handler->handle_entries_available(); + + wait_for_notification(); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, replay_ctx.wait()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +TEST_F(TestMockImageReplayerJournalReplayer, ImageNameUpdated) { + librbd::MockTestJournal mock_local_journal; + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx, + mock_local_journal}; + ::journal::MockJournaler mock_remote_journaler; + MockReplayerListener mock_replayer_listener; + MockThreads mock_threads{m_threads}; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_journaler, + {}); + MockReplayer mock_replayer{ + &mock_threads, "local mirror uuid", &mock_state_builder, + &mock_replayer_listener}; + + ::journal::MockReplayEntry mock_replay_entry; + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + expect_committed(mock_replay_entry, mock_remote_journaler, 1); + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockReplay mock_local_journal_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + librbd::journal::Listener* local_journal_listener = nullptr; + ::journal::ReplayHandler* remote_replay_handler = nullptr; + ::journal::JournalMetadataListener* remote_journaler_listener = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_replayer_listener, mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay, + &local_journal_listener, + &remote_replay_handler, + &remote_journaler_listener)); + + mock_local_image_ctx.name = "NEW NAME"; + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + expect_shut_down(mock_local_journal_replay, false, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_journal_replay, + 0); + expect_local_journal_add_listener(mock_local_journal, + &local_journal_listener); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + EXPECT_CALL(mock_local_journal_replay, decode(_, _)).WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, false, 0); + expect_process(mock_local_journal_replay, 0, 0); + EXPECT_CALL(mock_replay_status_formatter, handle_entry_processed(_)); + + // attempt to process the next event + C_SaferCond replay_ctx; + expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx); + + remote_replay_handler->handle_entries_available(); + wait_for_notification(); + + auto image_spec = util::compute_image_spec(m_local_io_ctx, "NEW NAME"); + ASSERT_EQ(image_spec, mock_replayer.get_image_spec()); + + ASSERT_EQ(0, replay_ctx.wait()); + ASSERT_TRUE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_journal, + mock_remote_journaler, + mock_local_journal_replay)); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_ApplyImageStateRequest.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_ApplyImageStateRequest.cc new file mode 100644 index 000000000..904d36854 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_ApplyImageStateRequest.cc @@ -0,0 +1,641 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/internal.h" +#include "librbd/Operations.h" +#include "librbd/image/GetMetadataRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace image { + +template <> +struct GetMetadataRequest<MockTestImageCtx> { + std::map<std::string, bufferlist>* pairs = nullptr; + Context* on_finish = nullptr; + + static GetMetadataRequest* s_instance; + static GetMetadataRequest* create(librados::IoCtx& io_ctx, + const std::string& oid, + bool filter_internal, + const std::string& filter_key_prefix, + const std::string& last_key, + size_t max_results, + std::map<std::string, bufferlist>* pairs, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->pairs = pairs; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetMetadataRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +GetMetadataRequest<MockTestImageCtx>* GetMetadataRequest<MockTestImageCtx>::s_instance = nullptr; + +} // namespace image +} // namespace librbd + +#include "tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.cc" + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +class TestMockImageReplayerSnapshotApplyImageStateRequest : public TestMockFixture { +public: + typedef ApplyImageStateRequest<librbd::MockTestImageCtx> MockApplyImageStateRequest; + typedef librbd::image::GetMetadataRequest<librbd::MockTestImageCtx> MockGetMetadataRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + + m_mock_local_image_ctx = new librbd::MockTestImageCtx(*m_local_image_ctx); + m_mock_remote_image_ctx = new librbd::MockTestImageCtx(*m_remote_image_ctx); + } + + void TearDown() override { + delete m_mock_remote_image_ctx; + delete m_mock_local_image_ctx; + TestMockFixture::TearDown(); + } + + void expect_rename_image(const std::string& name, int r) { + EXPECT_CALL(*m_mock_local_image_ctx->operations, execute_rename(name, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_update_features(uint64_t features, bool enable, int r) { + EXPECT_CALL(*m_mock_local_image_ctx->operations, + execute_update_features(features, enable, _, 0U)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_get_metadata(MockGetMetadataRequest& mock_get_metadata_request, + const std::map<std::string, bufferlist>& pairs, + int r) { + EXPECT_CALL(mock_get_metadata_request, send()) + .WillOnce(Invoke([this, &mock_get_metadata_request, pairs, r]() { + *mock_get_metadata_request.pairs = pairs; + m_threads->work_queue->queue(mock_get_metadata_request.on_finish, r); + })); + } + + void expect_update_metadata(const std::vector<std::string>& remove, + const std::map<std::string, bufferlist>& pairs, + int r) { + for (auto& key : remove) { + bufferlist bl; + ceph::encode(key, bl); + EXPECT_CALL(get_mock_io_ctx(m_mock_local_image_ctx->md_ctx), + exec(m_mock_local_image_ctx->header_oid, _, StrEq("rbd"), + StrEq("metadata_remove"), ContentsEqual(bl), _, _, _)) + .WillOnce(Return(r)); + if (r < 0) { + return; + } + } + + if (!pairs.empty()) { + bufferlist bl; + ceph::encode(pairs, bl); + EXPECT_CALL(get_mock_io_ctx(m_mock_local_image_ctx->md_ctx), + exec(m_mock_local_image_ctx->header_oid, _, StrEq("rbd"), + StrEq("metadata_set"), ContentsEqual(bl), _, _, _)) + .WillOnce(Return(r)); + } + } + + void expect_unprotect_snapshot(const std::string& name, int r) { + EXPECT_CALL(*m_mock_local_image_ctx->operations, + execute_snap_unprotect({cls::rbd::UserSnapshotNamespace{}}, + name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_remove_snapshot(const std::string& name, int r) { + EXPECT_CALL(*m_mock_local_image_ctx->operations, + execute_snap_remove({cls::rbd::UserSnapshotNamespace{}}, + name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_protect_snapshot(const std::string& name, int r) { + EXPECT_CALL(*m_mock_local_image_ctx->operations, + execute_snap_protect({cls::rbd::UserSnapshotNamespace{}}, + name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_rename_snapshot(uint64_t snap_id, const std::string& name, + int r) { + EXPECT_CALL(*m_mock_local_image_ctx->operations, + execute_snap_rename(snap_id, name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + + void expect_set_snap_limit(uint64_t limit, int r) { + EXPECT_CALL(*m_mock_local_image_ctx->operations, + execute_snap_set_limit(limit, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + librbd::ImageCtx *m_local_image_ctx; + librbd::ImageCtx *m_remote_image_ctx; + + librbd::MockTestImageCtx *m_mock_local_image_ctx = nullptr; + librbd::MockTestImageCtx *m_mock_remote_image_ctx = nullptr; + +}; + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, NoChanges) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, RenameImage) { + InSequence seq; + + expect_rename_image("new name", 0); + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = "new name"; + image_state.features = m_remote_image_ctx->features; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, RenameImageError) { + InSequence seq; + + expect_rename_image("new name", -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = "new name"; + image_state.features = m_remote_image_ctx->features; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, UpdateFeatures) { + InSequence seq; + + expect_update_features(RBD_FEATURE_DEEP_FLATTEN, false, 0); + expect_update_features(RBD_FEATURE_OBJECT_MAP, true, 0); + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = RBD_FEATURE_EXCLUSIVE_LOCK | + RBD_FEATURE_OBJECT_MAP; + m_mock_local_image_ctx->features = RBD_FEATURE_EXCLUSIVE_LOCK | + RBD_FEATURE_DEEP_FLATTEN; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, UpdateFeaturesError) { + InSequence seq; + + expect_update_features(RBD_FEATURE_DEEP_FLATTEN, false, -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = RBD_FEATURE_EXCLUSIVE_LOCK | + RBD_FEATURE_OBJECT_MAP; + m_mock_local_image_ctx->features = RBD_FEATURE_EXCLUSIVE_LOCK | + RBD_FEATURE_DEEP_FLATTEN; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, UpdateImageMeta) { + InSequence seq; + + bufferlist data_bl; + ceph::encode("data", data_bl); + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, + {{"key1", {}}, {"key2", {}}}, 0); + expect_update_metadata({"key2"}, {{"key1", data_bl}}, 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.metadata = {{"key1", data_bl}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, GetImageMetaError) { + InSequence seq; + + bufferlist data_bl; + ceph::encode("data", data_bl); + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, + {{"key1", {}}, {"key2", {}}}, -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.metadata = {{"key1", data_bl}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, UpdateImageMetaError) { + InSequence seq; + + bufferlist data_bl; + ceph::encode("data", data_bl); + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, + {{"key1", {}}, {"key2", {}}}, 0); + expect_update_metadata({"key2"}, {{"key1", data_bl}}, -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.metadata = {{"key1", data_bl}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, UnprotectSnapshot) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_unprotect_snapshot("snap1", 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.snapshots = { + {1U, {cls::rbd::UserSnapshotNamespace{}, "snap1", + RBD_PROTECTION_STATUS_UNPROTECTED}}}; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, UnprotectSnapshotError) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_unprotect_snapshot("snap1", -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.snapshots = { + {1U, {cls::rbd::UserSnapshotNamespace{}, "snap1", + RBD_PROTECTION_STATUS_UNPROTECTED}}}; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, RemoveSnapshot) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_remove_snapshot("snap1", 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, RemoveSnapshotError) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_remove_snapshot("snap1", -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, ProtectSnapshot) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_protect_snapshot("snap1", 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.snapshots = { + {1U, {cls::rbd::UserSnapshotNamespace{}, "snap1", + RBD_PROTECTION_STATUS_PROTECTED}}}; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, ProtectSnapshotError) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_protect_snapshot("snap1", -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.snapshots = { + {1U, {cls::rbd::UserSnapshotNamespace{}, "snap1", + RBD_PROTECTION_STATUS_PROTECTED}}}; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, RenameSnapshot) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_rename_snapshot(11, "snap1-renamed", 0); + + expect_set_snap_limit(0, 0); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.snapshots = { + {1U, {cls::rbd::UserSnapshotNamespace{}, "snap1-renamed", + RBD_PROTECTION_STATUS_PROTECTED}}}; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, RenameSnapshotError) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_rename_snapshot(11, "snap1-renamed", -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + image_state.snapshots = { + {1U, {cls::rbd::UserSnapshotNamespace{}, "snap1-renamed", + RBD_PROTECTION_STATUS_PROTECTED}}}; + m_mock_local_image_ctx->snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0U, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, 11}, {2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotApplyImageStateRequest, SetSnapshotLimitError) { + InSequence seq; + + MockGetMetadataRequest mock_get_metadata_request; + expect_get_metadata(mock_get_metadata_request, {}, 0); + + expect_set_snap_limit(0, -EINVAL); + + librbd::mirror::snapshot::ImageState image_state; + image_state.name = m_image_name; + image_state.features = m_remote_image_ctx->features; + + C_SaferCond ctx; + auto req = MockApplyImageStateRequest::create( + "local mirror uuid", "remote mirror uuid", m_mock_local_image_ctx, + m_mock_remote_image_ctx, image_state, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc new file mode 100644 index 000000000..58214f3e8 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_CreateLocalImageRequest.cc @@ -0,0 +1,356 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/internal.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include <boost/intrusive_ptr.hpp> + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace util { + +static std::string s_image_id; + +template <> +std::string generate_image_id<MockTestImageCtx>(librados::IoCtx&) { + ceph_assert(!s_image_id.empty()); + return s_image_id; +} + +} // namespace util +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +namespace image_replayer { + +template<> +struct CreateImageRequest<librbd::MockTestImageCtx> { + static CreateImageRequest* s_instance; + Context *on_finish = nullptr; + + static CreateImageRequest* create(Threads<librbd::MockTestImageCtx>* threads, + librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + librbd::MockTestImageCtx *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + s_instance->construct(local_image_id); + return s_instance; + } + + CreateImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~CreateImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); +}; + +CreateImageRequest<librbd::MockTestImageCtx>* + CreateImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +namespace snapshot { + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + std::string local_image_id; + std::string remote_mirror_uuid; +}; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc" + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +class TestMockImageReplayerSnapshotCreateLocalImageRequest : public TestMockFixture { +public: + typedef CreateLocalImageRequest<librbd::MockTestImageCtx> MockCreateLocalImageRequest; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef CreateImageRequest<librbd::MockTestImageCtx> MockCreateImageRequest; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + m_mock_remote_image_ctx = new librbd::MockTestImageCtx(*m_remote_image_ctx); + } + + void TearDown() override { + delete m_mock_remote_image_ctx; + TestMockFixture::TearDown(); + } + + void snap_create(librbd::ImageCtx *image_ctx, const std::string &snap_name) { + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, image_ctx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + snap_name, 0, prog_ctx)); + ASSERT_EQ(0, image_ctx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + snap_name)); + ASSERT_EQ(0, image_ctx->state->refresh()); + } + + int clone_image(librbd::ImageCtx *parent_image_ctx, + const std::string &snap_name, const std::string &clone_name) { + snap_create(parent_image_ctx, snap_name); + + int order = 0; + return librbd::clone(m_remote_io_ctx, parent_image_ctx->name.c_str(), + snap_name.c_str(), m_remote_io_ctx, + clone_name.c_str(), parent_image_ctx->features, + &order, 0, 0); + } + + void expect_mirror_image_set(const std::string& image_id, + const cls::rbd::MirrorImage& mirror_image, + int r) { + bufferlist bl; + encode(image_id, bl); + encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_set"), ContentsEqual(bl), _, _, _)) + .WillOnce(Return(r)); + } + + void expect_mirror_image_remove(const std::string& image_id, int r) { + bufferlist bl; + encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(StrEq("rbd_mirroring"), _, StrEq("rbd"), + StrEq("mirror_image_remove"), + ContentsEqual(bl), _, _, _)) + .WillOnce(Return(r)); + } + + void expect_create_image(MockCreateImageRequest& mock_create_image_request, + const std::string& image_id, int r) { + EXPECT_CALL(mock_create_image_request, construct(image_id)); + EXPECT_CALL(mock_create_image_request, send()) + .WillOnce(Invoke([this, &mock_create_image_request, r]() { + m_threads->work_queue->queue(mock_create_image_request.on_finish, r); + })); + } + + MockCreateLocalImageRequest* create_request( + MockThreads& mock_threads, + MockStateBuilder& mock_state_builder, + const std::string& global_image_id, + Context* on_finish) { + return new MockCreateLocalImageRequest( + &mock_threads, m_local_io_ctx, m_mock_remote_image_ctx, + global_image_id, &m_pool_meta_cache, nullptr, &mock_state_builder, + on_finish); + } + + PoolMetaCache m_pool_meta_cache{g_ceph_context}; + + librbd::ImageCtx *m_remote_image_ctx; + librbd::MockTestImageCtx *m_mock_remote_image_ctx = nullptr; +}; + +TEST_F(TestMockImageReplayerSnapshotCreateLocalImageRequest, Success) { + InSequence seq; + + librbd::util::s_image_id = "local image id"; + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_CREATING}, 0); + + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, "local image id", 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ("local image id", mock_state_builder.local_image_id); +} + +TEST_F(TestMockImageReplayerSnapshotCreateLocalImageRequest, AddMirrorImageError) { + InSequence seq; + + librbd::util::s_image_id = "local image id"; + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_CREATING}, -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotCreateLocalImageRequest, CreateImageError) { + InSequence seq; + + librbd::util::s_image_id = "local image id"; + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_CREATING}, 0); + + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, "local image id", -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotCreateLocalImageRequest, CreateImageDuplicate) { + InSequence seq; + + librbd::util::s_image_id = "local image id"; + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_CREATING}, 0); + + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, "local image id", -EBADF); + + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_mirror_image_remove("local image id", 0); + + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_CREATING}, 0); + + expect_create_image(mock_create_image_request, "local image id", 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockStateBuilder mock_state_builder; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ("local image id", mock_state_builder.local_image_id); +} + +TEST_F(TestMockImageReplayerSnapshotCreateLocalImageRequest, DisableMirrorImageError) { + InSequence seq; + + librbd::util::s_image_id = "local image id"; + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockStateBuilder mock_state_builder; + mock_state_builder.local_image_id = "local image id"; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotCreateLocalImageRequest, RemoveMirrorImageError) { + InSequence seq; + + librbd::util::s_image_id = "local image id"; + expect_mirror_image_set("local image id", + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, 0); + + expect_mirror_image_remove("local image id", -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockStateBuilder mock_state_builder; + mock_state_builder.local_image_id = "local image id"; + auto request = create_request( + mock_threads, mock_state_builder, "global image id", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc new file mode 100644 index 000000000..e82381b09 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc @@ -0,0 +1,3320 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/deep_copy/ImageCopyRequest.h" +#include "librbd/deep_copy/SnapshotCopyRequest.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "librbd/mirror/snapshot/GetImageStateRequest.h" +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/ReplayerListener.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/Replayer.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockOperations.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace deep_copy { + +template <> +struct ImageCopyRequest<MockTestImageCtx> { + uint64_t src_snap_id_start; + uint64_t src_snap_id_end; + uint64_t dst_snap_id_start; + librbd::deep_copy::ObjectNumber object_number; + librbd::SnapSeqs snap_seqs; + + static ImageCopyRequest* s_instance; + static ImageCopyRequest* create(MockTestImageCtx *src_image_ctx, + MockTestImageCtx *dst_image_ctx, + librados::snap_t src_snap_id_start, + librados::snap_t src_snap_id_end, + librados::snap_t dst_snap_id_start, + bool flatten, + const ObjectNumber &object_number, + const SnapSeqs &snap_seqs, + Handler *handler, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->src_snap_id_start = src_snap_id_start; + s_instance->src_snap_id_end = src_snap_id_end; + s_instance->dst_snap_id_start = dst_snap_id_start; + s_instance->object_number = object_number; + s_instance->snap_seqs = snap_seqs; + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + + ImageCopyRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template <> +struct SnapshotCopyRequest<MockTestImageCtx> { + librados::snap_t src_snap_id_start; + librados::snap_t src_snap_id_end; + librados::snap_t dst_snap_id_start; + SnapSeqs* snap_seqs = nullptr; + + static SnapshotCopyRequest* s_instance; + static SnapshotCopyRequest* create(MockTestImageCtx *src_image_ctx, + MockTestImageCtx *dst_image_ctx, + librados::snap_t src_snap_id_start, + librados::snap_t src_snap_id_end, + librados::snap_t dst_snap_id_start, + bool flatten, + ::MockContextWQ *work_queue, + SnapSeqs *snap_seqs, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->src_snap_id_start = src_snap_id_start; + s_instance->src_snap_id_end = src_snap_id_end; + s_instance->dst_snap_id_start = dst_snap_id_start; + s_instance->snap_seqs = snap_seqs; + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + + SnapshotCopyRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +ImageCopyRequest<MockTestImageCtx>* ImageCopyRequest<MockTestImageCtx>::s_instance = nullptr; +SnapshotCopyRequest<MockTestImageCtx>* SnapshotCopyRequest<MockTestImageCtx>::s_instance = nullptr; + +} // namespace deep_copy + +namespace mirror { + +template <> +struct ImageStateUpdateRequest<MockTestImageCtx> { + static ImageStateUpdateRequest* s_instance; + static ImageStateUpdateRequest* create( + librados::IoCtx& io_ctx, + const std::string& image_id, + cls::rbd::MirrorImageState mirror_image_state, + const cls::rbd::MirrorImage& mirror_image, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_EQ(cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + mirror_image_state); + EXPECT_EQ(cls::rbd::MirrorImage{}, mirror_image); + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + ImageStateUpdateRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +ImageStateUpdateRequest<MockTestImageCtx>* ImageStateUpdateRequest<MockTestImageCtx>::s_instance = nullptr; + +namespace snapshot { + +template <> +struct CreateNonPrimaryRequest<MockTestImageCtx> { + bool demoted = false; + std::string primary_mirror_uuid; + uint64_t primary_snap_id; + SnapSeqs snap_seqs; + uint64_t* snap_id = nullptr; + + static CreateNonPrimaryRequest* s_instance; + static CreateNonPrimaryRequest* create(MockTestImageCtx *image_ctx, + bool demoted, + const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, + const SnapSeqs& snap_seqs, + const ImageState &image_state, + uint64_t *snap_id, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->demoted = demoted; + s_instance->primary_mirror_uuid = primary_mirror_uuid; + s_instance->primary_snap_id = primary_snap_id; + s_instance->snap_seqs = snap_seqs; + s_instance->snap_id = snap_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + + CreateNonPrimaryRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template <> +struct GetImageStateRequest<MockTestImageCtx> { + uint64_t snap_id = CEPH_NOSNAP; + + static GetImageStateRequest* s_instance; + static GetImageStateRequest* create(MockTestImageCtx *image_ctx, + uint64_t snap_id, + ImageState *image_state, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->snap_id = snap_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + + GetImageStateRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template <> +struct ImageMeta<MockTestImageCtx> { + MOCK_METHOD1(load, void(Context*)); + + bool resync_requested = false; +}; + +template <> +struct UnlinkPeerRequest<MockTestImageCtx> { + uint64_t snap_id; + std::string mirror_peer_uuid; + + static UnlinkPeerRequest* s_instance; + static UnlinkPeerRequest*create (MockTestImageCtx *image_ctx, + uint64_t snap_id, + const std::string &mirror_peer_uuid, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->snap_id = snap_id; + s_instance->mirror_peer_uuid = mirror_peer_uuid; + s_instance->on_finish = on_finish; + return s_instance; + } + + Context* on_finish = nullptr; + + UnlinkPeerRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +CreateNonPrimaryRequest<MockTestImageCtx>* CreateNonPrimaryRequest<MockTestImageCtx>::s_instance = nullptr; +GetImageStateRequest<MockTestImageCtx>* GetImageStateRequest<MockTestImageCtx>::s_instance = nullptr; +UnlinkPeerRequest<MockTestImageCtx>* UnlinkPeerRequest<MockTestImageCtx>::s_instance = nullptr; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct InstanceWatcher<librbd::MockTestImageCtx> { + MOCK_METHOD1(cancel_sync_request, void(const std::string&)); + MOCK_METHOD2(notify_sync_request, void(const std::string&, + Context*)); + MOCK_METHOD1(notify_sync_complete, void(const std::string&)); +}; + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx>* threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +namespace { + +struct MockReplayerListener : public image_replayer::ReplayerListener { + MOCK_METHOD0(handle_notification, void()); +}; + +} // anonymous namespace + +namespace image_replayer { + +template<> +struct CloseImageRequest<librbd::MockTestImageCtx> { + static CloseImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + return s_instance; + } + + CloseImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~CloseImageRequest() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +namespace snapshot { + +template <> +struct ApplyImageStateRequest<librbd::MockTestImageCtx> { + Context* on_finish = nullptr; + + static ApplyImageStateRequest* s_instance; + static ApplyImageStateRequest* create( + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + librbd::MockTestImageCtx* local_image_ctx, + librbd::MockTestImageCtx* remote_image_ctx, + const librbd::mirror::snapshot::ImageState& image_state, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + ApplyImageStateRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + StateBuilder(librbd::MockTestImageCtx& local_image_ctx, + librbd::MockTestImageCtx& remote_image_ctx, + librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx>& + local_image_meta) + : local_image_ctx(&local_image_ctx), + remote_image_ctx(&remote_image_ctx), + local_image_meta(&local_image_meta) { + } + + librbd::MockTestImageCtx* local_image_ctx; + librbd::MockTestImageCtx* remote_image_ctx; + + std::string remote_mirror_uuid = "remote mirror uuid"; + + librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx>* + local_image_meta = nullptr; +}; + +ApplyImageStateRequest<librbd::MockTestImageCtx>* ApplyImageStateRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_replayer/snapshot/Replayer.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageReplayerSnapshotReplayer : public TestMockFixture { +public: + typedef Replayer<librbd::MockTestImageCtx> MockReplayer; + typedef ApplyImageStateRequest<librbd::MockTestImageCtx> MockApplyImageStateRequest; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest; + typedef librbd::deep_copy::ImageCopyRequest<librbd::MockTestImageCtx> MockImageCopyRequest; + typedef librbd::deep_copy::SnapshotCopyRequest<librbd::MockTestImageCtx> MockSnapshotCopyRequest; + typedef librbd::mirror::ImageStateUpdateRequest<librbd::MockTestImageCtx> MockImageStateUpdateRequest; + typedef librbd::mirror::snapshot::CreateNonPrimaryRequest<librbd::MockTestImageCtx> MockCreateNonPrimaryRequest; + typedef librbd::mirror::snapshot::GetImageStateRequest<librbd::MockTestImageCtx> MockGetImageStateRequest; + typedef librbd::mirror::snapshot::ImageMeta<librbd::MockTestImageCtx> MockImageMeta; + typedef librbd::mirror::snapshot::UnlinkPeerRequest<librbd::MockTestImageCtx> MockUnlinkPeerRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, + m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, + &m_remote_image_ctx)); + } + + void expect_work_queue_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event_after_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillRepeatedly( + DoAll(Invoke([this](double seconds, Context *ctx) { + m_threads->timer->add_event_after(seconds, ctx); + }), + ReturnArg<1>())); + EXPECT_CALL(*mock_threads.timer, cancel_event(_)) + .WillRepeatedly( + Invoke([this](Context *ctx) { + return m_threads->timer->cancel_event(ctx); + })); + } + + void expect_register_update_watcher(librbd::MockTestImageCtx& mock_image_ctx, + librbd::UpdateWatchCtx** update_watch_ctx, + uint64_t watch_handle, int r) { + EXPECT_CALL(*mock_image_ctx.state, register_update_watcher(_, _)) + .WillOnce(Invoke([update_watch_ctx, watch_handle, r] + (librbd::UpdateWatchCtx* ctx, uint64_t* handle) { + if (r >= 0) { + *update_watch_ctx = ctx; + *handle = watch_handle; + } + return r; + })); + } + + void expect_unregister_update_watcher(librbd::MockTestImageCtx& mock_image_ctx, + uint64_t watch_handle, int r) { + EXPECT_CALL(*mock_image_ctx.state, unregister_update_watcher(watch_handle, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_load_image_meta(MockImageMeta& mock_image_meta, + bool resync_requested, int r) { + EXPECT_CALL(mock_image_meta, load(_)) + .WillOnce(Invoke([this, &mock_image_meta, resync_requested, r](Context* ctx) { + mock_image_meta.resync_requested = resync_requested; + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_is_refresh_required(librbd::MockTestImageCtx& mock_image_ctx, + bool is_required) { + EXPECT_CALL(*mock_image_ctx.state, is_refresh_required()) + .WillOnce(Return(is_required)); + } + + void expect_refresh(librbd::MockTestImageCtx& mock_image_ctx, + const std::map<uint64_t, librbd::SnapInfo>& snaps, + int r) { + EXPECT_CALL(*mock_image_ctx.state, refresh(_)) + .WillOnce(Invoke([this, &mock_image_ctx, snaps, r](Context* ctx) { + mock_image_ctx.snap_info = snaps; + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_notify_update(librbd::MockTestImageCtx& mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, notify_update(_)) + .WillOnce(Invoke([this](Context* ctx) { + m_threads->work_queue->queue(ctx, 0); + })); + } + + void expect_prune_non_primary_snapshot(librbd::MockTestImageCtx& mock_image_ctx, + uint64_t snap_id, int r) { + EXPECT_CALL(mock_image_ctx, get_snap_info(snap_id)) + .WillOnce(Invoke([&mock_image_ctx](uint64_t snap_id) -> librbd::SnapInfo* { + auto it = mock_image_ctx.snap_info.find(snap_id); + if (it == mock_image_ctx.snap_info.end()) { + return nullptr; + } + return &it->second; + })); + EXPECT_CALL(*mock_image_ctx.operations, snap_remove(_, _, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_snapshot_copy(MockSnapshotCopyRequest& mock_snapshot_copy_request, + uint64_t src_snap_id_start, + uint64_t src_snap_id_end, + uint64_t dst_snap_id_start, + const librbd::SnapSeqs& snap_seqs, int r) { + EXPECT_CALL(mock_snapshot_copy_request, send()) + .WillOnce(Invoke([this, &req=mock_snapshot_copy_request, + src_snap_id_start, src_snap_id_end, dst_snap_id_start, + snap_seqs, r]() { + ASSERT_EQ(src_snap_id_start, req.src_snap_id_start); + ASSERT_EQ(src_snap_id_end, req.src_snap_id_end); + ASSERT_EQ(dst_snap_id_start, req.dst_snap_id_start); + *req.snap_seqs = snap_seqs; + m_threads->work_queue->queue(req.on_finish, r); + })); + } + + void expect_get_image_state(MockGetImageStateRequest& mock_get_image_state_request, + uint64_t snap_id, int r) { + EXPECT_CALL(mock_get_image_state_request, send()) + .WillOnce(Invoke([this, &req=mock_get_image_state_request, snap_id, r]() { + ASSERT_EQ(snap_id, req.snap_id); + m_threads->work_queue->queue(req.on_finish, r); + })); + } + + void expect_create_non_primary_request(MockCreateNonPrimaryRequest& mock_create_non_primary_request, + bool demoted, + const std::string& primary_mirror_uuid, + uint64_t primary_snap_id, + const librbd::SnapSeqs& snap_seqs, + uint64_t snap_id, int r) { + EXPECT_CALL(mock_create_non_primary_request, send()) + .WillOnce(Invoke([this, &req=mock_create_non_primary_request, demoted, + primary_mirror_uuid, primary_snap_id, snap_seqs, + snap_id, r]() { + ASSERT_EQ(demoted, req.demoted); + ASSERT_EQ(primary_mirror_uuid, req.primary_mirror_uuid); + ASSERT_EQ(primary_snap_id, req.primary_snap_id); + ASSERT_EQ(snap_seqs, req.snap_seqs); + *req.snap_id = snap_id; + m_threads->work_queue->queue(req.on_finish, r); + })); + } + + void expect_update_mirror_image_state(MockImageStateUpdateRequest& mock_image_state_update_request, + int r) { + EXPECT_CALL(mock_image_state_update_request, send()) + .WillOnce(Invoke([this, &req=mock_image_state_update_request, r]() { + m_threads->work_queue->queue(req.on_finish, r); + })); + } + + void expect_notify_sync_request(MockInstanceWatcher& mock_instance_watcher, + const std::string& image_id, int r) { + EXPECT_CALL(mock_instance_watcher, notify_sync_request(image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_notify_sync_complete(MockInstanceWatcher& mock_instance_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_instance_watcher, notify_sync_complete(image_id)); + } + + void expect_cancel_sync_request(MockInstanceWatcher& mock_instance_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_instance_watcher, cancel_sync_request(image_id)); + } + + void expect_image_copy(MockImageCopyRequest& mock_image_copy_request, + uint64_t src_snap_id_start, uint64_t src_snap_id_end, + uint64_t dst_snap_id_start, + const librbd::deep_copy::ObjectNumber& object_number, + const librbd::SnapSeqs& snap_seqs, int r) { + EXPECT_CALL(mock_image_copy_request, send()) + .WillOnce(Invoke([this, &req=mock_image_copy_request, src_snap_id_start, + src_snap_id_end, dst_snap_id_start, object_number, + snap_seqs, r]() { + ASSERT_EQ(src_snap_id_start, req.src_snap_id_start); + ASSERT_EQ(src_snap_id_end, req.src_snap_id_end); + ASSERT_EQ(dst_snap_id_start, req.dst_snap_id_start); + ASSERT_EQ(object_number, req.object_number); + ASSERT_EQ(snap_seqs, req.snap_seqs); + m_threads->work_queue->queue(req.on_finish, r); + })); + } + + void expect_unlink_peer(MockUnlinkPeerRequest& mock_unlink_peer_request, + uint64_t snap_id, const std::string& mirror_peer_uuid, + int r) { + EXPECT_CALL(mock_unlink_peer_request, send()) + .WillOnce(Invoke([this, &req=mock_unlink_peer_request, snap_id, + mirror_peer_uuid, r]() { + ASSERT_EQ(snap_id, req.snap_id); + ASSERT_EQ(mirror_peer_uuid, req.mirror_peer_uuid); + m_threads->work_queue->queue(req.on_finish, r); + })); + } + + void expect_apply_image_state( + MockApplyImageStateRequest& mock_request, int r) { + EXPECT_CALL(mock_request, send()) + .WillOnce(Invoke([this, &req=mock_request, r]() { + m_threads->work_queue->queue(req.on_finish, r); + })); + } + + void expect_mirror_image_snapshot_set_copy_progress( + librbd::MockTestImageCtx& mock_test_image_ctx, uint64_t snap_id, + bool completed, uint64_t last_copied_object, int r) { + bufferlist bl; + encode(snap_id, bl); + encode(completed, bl); + encode(last_copied_object, bl); + + EXPECT_CALL(get_mock_io_ctx(mock_test_image_ctx.md_ctx), + exec(mock_test_image_ctx.header_oid, _, StrEq("rbd"), + StrEq("mirror_image_snapshot_set_copy_progress"), + ContentsEqual(bl), _, _, _)) + .WillOnce(Return(r)); + } + + void expect_send(MockCloseImageRequest &mock_close_image_request, int r) { + EXPECT_CALL(mock_close_image_request, send()) + .WillOnce(Invoke([this, &mock_close_image_request, r]() { + *mock_close_image_request.image_ctx = nullptr; + m_threads->work_queue->queue(mock_close_image_request.on_finish, r); + })); + } + + void expect_notification(MockThreads& mock_threads, + MockReplayerListener& mock_replayer_listener) { + EXPECT_CALL(mock_replayer_listener, handle_notification()) + .WillRepeatedly(Invoke([this]() { + std::unique_lock locker{m_lock}; + ++m_notifications; + m_cond.notify_all(); + })); + } + + int wait_for_notification(uint32_t count) { + std::unique_lock locker{m_lock}; + for (uint32_t idx = 0; idx < count; ++idx) { + while (m_notifications == 0) { + if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) { + return -ETIMEDOUT; + } + } + --m_notifications; + } + return 0; + } + + int init_entry_replayer(MockReplayer& mock_replayer, + MockThreads& mock_threads, + librbd::MockTestImageCtx& mock_local_image_ctx, + librbd::MockTestImageCtx& mock_remote_image_ctx, + MockReplayerListener& mock_replayer_listener, + MockImageMeta& mock_image_meta, + librbd::UpdateWatchCtx** update_watch_ctx) { + expect_register_update_watcher(mock_local_image_ctx, update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, update_watch_ctx, 234, + 0); + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + int r = init_ctx.wait(); + if (r < 0) { + return r; + } + + return wait_for_notification(2); + } + + int shut_down_entry_replayer(MockReplayer& mock_replayer, + MockThreads& mock_threads, + librbd::MockTestImageCtx& mock_local_image_ctx, + librbd::MockTestImageCtx& mock_remote_image_ctx) { + expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + return shutdown_ctx.wait(); + } + + librbd::ImageCtx* m_local_image_ctx = nullptr; + librbd::ImageCtx* m_remote_image_ctx = nullptr; + + PoolMetaCache m_pool_meta_cache{g_ceph_context}; + + ceph::mutex m_lock = ceph::make_mutex( + "TestMockImageReplayerSnapshotReplayer"); + ceph::condition_variable m_cond; + uint32_t m_notifications = 0; +}; + +TEST_F(TestMockImageReplayerSnapshotReplayer, InitShutDown) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + // it should sync two snapshots and skip two (user and mirror w/o matching + // peer uuid) + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + + // init + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, + 0); + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // sync snap4 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {5U, librbd::SnapInfo{"snap5", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); + expect_snapshot_copy(mock_snapshot_copy_request, 1, 4, 11, + {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 0); + expect_get_image_state(mock_get_image_state_request, 4, 0); + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 4, + {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 14, + 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_image_copy(mock_image_copy_request, 1, 4, 11, {}, + {{1, 11}, {2, 12}, {4, CEPH_NOSNAP}}, 0); + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 14, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // prune non-primary snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 4, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 4, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); + + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(4)); + + // shut down + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncInitial) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject an incomplete sync snapshot with last_copied_object_number > 0 + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, false, 123, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // re-sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, + librbd::deep_copy::ObjectNumber{123U}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 123, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncDelta) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject an incomplete sync snapshot with last_copied_object_number > 0 + // after a complete snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 123, {{2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // re-sync snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 12, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, + librbd::deep_copy::ObjectNumber{123U}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 12, true, 123, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // prune non-primary snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSyncDeltaDemote) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject an incomplete sync snapshot with last_copied_object_number > 0 + // after a primary demotion snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 11, true, 0, + {{11, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 123, {{2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // re-sync snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 12, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, + librbd::deep_copy::ObjectNumber{123U}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 12, true, 123, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncInitial) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject an incomplete sync snapshot with last_copied_object_number == 0 + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, false, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // re-sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncDelta) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject an incomplete sync snapshot with last_copied_object_number == 0 + // after a complete snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 0, {{2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // prune non-primary snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 12, 0); + + // sync snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, + {{2, CEPH_NOSNAP}}, 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 2, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 2, + {{2, CEPH_NOSNAP}}, 13, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 13, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // prune non-primary snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 11, 0); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedPendingSyncDeltaDemote) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject an incomplete sync snapshot with last_copied_object_number == 0 + // after a primary demotion snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 11, true, 0, + {{11, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, false, 0, {{2, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + + // prune non-primary snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + expect_prune_non_primary_snapshot(mock_local_image_ctx, 12, 0); + + // sync snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, + {{2, CEPH_NOSNAP}}, 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 2, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 2, + {{2, CEPH_NOSNAP}}, 13, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 13, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {13U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 2, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(2)); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a demotion snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + true, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(2)); + ASSERT_FALSE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a promotion snapshot + mock_local_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, + {"remote mirror peer uuid"}, "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // idle + expect_load_image_meta(mock_image_meta, true, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterLocalUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayerListener mock_replayer_listener; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + // init + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + -EINVAL); + + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-EINVAL, init_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayerListener mock_replayer_listener; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + // init + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, + -EINVAL); + + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(-EINVAL, init_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + + // shut down + expect_unregister_update_watcher(mock_remote_image_ctx, 234, -EINVAL); + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + + // shut down + expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); + expect_unregister_update_watcher(mock_local_image_ctx, 123, -EINVAL); + + C_SaferCond shutdown_ctx; + mock_replayer.shut_down(&shutdown_ctx); + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // sync + expect_load_image_meta(mock_image_meta, false, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // sync + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh(mock_local_image_ctx, {}, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // sync + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh(mock_remote_image_ctx, {}, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, -EINVAL); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateMirrorImageStateError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, -EIO); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EIO, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, + -ECANCELED); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-ECANCELED, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP,true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, -EINVAL); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, -EINVAL); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap2 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 1, 2, 11, {{2, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 2, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 2, + {{2, CEPH_NOSNAP}}, 12, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, + {{2, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 12, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + -EINVAL); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EINVAL, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a primary demote to local image + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP, + true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // detect split-brain + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EEXIST, mock_replayer.get_error_code()); + ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteSnapshotMissingSplitBrain) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a missing remote start snap (deleted) + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, + "remote mirror uuid", 1, true, 0, + {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}}; + mock_remote_image_ctx.snap_info = { + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // split-brain due to missing snapshot 1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-EEXIST, mock_replayer.get_error_code()); + ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteFailover) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject a primary demote to local image + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 12U, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_local_image_ctx.snap_ids = { + {{cls::rbd::UserSnapshotNamespace{}, "snap1"}, 11}, + {{cls::rbd::MirrorSnapshotNamespace{}, "snap2"}, 12}}; + mock_local_image_ctx.snap_info = { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP, + true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // attach to promoted remote image + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 2, 3, 12, + {{2, 12}, {3, CEPH_NOSNAP}}, 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 3, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 3, + {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}, 13, + 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 2, 3, 12, {}, + {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 13, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 2, "remote mirror peer uuid", 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {12U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", CEPH_NOSNAP, + true, 0, {}}, + 0, {}, 0, 0, {}}}, + {13U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, + "remote mirror uuid", 3, true, 0, + {{1, 11}, {2, 12}, {3, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {1U, librbd::SnapInfo{"snap1", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {2U, librbd::SnapInfo{"snap2", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED, + {"remote mirror peer uuid"}, "local mirror uuid", 12U, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {}, "", CEPH_NOSNAP, true, 0, + {}}, + 0, {}, 0, 0, {}}} + }, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(2)); + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkRemoteSnapshot) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + // it should attempt to unlink from remote snap1 since we don't need it + // anymore + mock_local_image_ctx.snap_info = { + {14U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 4, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + + // init + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, + 0); + + // unlink snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockUnlinkPeerRequest mock_unlink_peer_request; + expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", + 0); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, true); + expect_refresh( + mock_remote_image_ctx, { + {2U, librbd::SnapInfo{"snap2", cls::rbd::UserSnapshotNamespace{}, + 0, {}, 0, 0, {}}}, + {3U, librbd::SnapInfo{"snap3", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {""}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}, + {4U, librbd::SnapInfo{"snap4", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}} + }, 0); + + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(3)); + + // shut down + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, SkipImageSync) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, + "", 0U, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + + // init + expect_register_update_watcher(mock_local_image_ctx, &update_watch_ctx, 123, + 0); + expect_register_update_watcher(mock_remote_image_ctx, &update_watch_ctx, 234, + 0); + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + MockApplyImageStateRequest mock_apply_state_request; + expect_apply_image_state(mock_apply_state_request, 0); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + + // idle + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, true); + expect_refresh( + mock_local_image_ctx, { + {11U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY, {}, "remote mirror uuid", + 1, true, 0, {{1, CEPH_NOSNAP}}}, + 0, {}, 0, 0, {}}}, + }, 0); + expect_is_refresh_required(mock_remote_image_ctx, false); + + // fire init + C_SaferCond init_ctx; + mock_replayer.init(&init_ctx); + ASSERT_EQ(0, init_ctx.wait()); + + // wait for sync to complete + ASSERT_EQ(0, wait_for_notification(3)); + + // shut down + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, ImageNameUpdated) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // change the name of the image + mock_local_image_ctx.name = "NEW NAME"; + + // idle + expect_load_image_meta(mock_image_meta, true, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(2)); + auto image_spec = image_replayer::util::compute_image_spec(m_local_io_ctx, + "NEW NAME"); + ASSERT_EQ(image_spec, mock_replayer.get_image_spec()); + ASSERT_FALSE(mock_replayer.is_replaying()); + + // shut down + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, ApplyImageStatePendingShutdown) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + C_SaferCond shutdown_ctx; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + EXPECT_CALL(mock_apply_state_request, send()) + .WillOnce(Invoke([this, &req=mock_apply_state_request, + &replayer=mock_replayer, &ctx=shutdown_ctx]() { + // inject a shutdown, to be pended due to STATE_REPLAYING + replayer.shut_down(&ctx); + m_threads->work_queue->queue(req.on_finish, 0); + })); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id); + expect_mirror_image_snapshot_set_copy_progress( + mock_local_image_ctx, 11, true, 0, 0); + expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // shutdown should be resumed + expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(0, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +TEST_F(TestMockImageReplayerSnapshotReplayer, ApplyImageStateErrorPendingShutdown) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + C_SaferCond shutdown_ctx; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + CEPH_NOSNAP, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + MockImageStateUpdateRequest mock_image_state_update_request; + expect_update_mirror_image_state(mock_image_state_update_request, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + MockImageCopyRequest mock_image_copy_request; + expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, + {{1, CEPH_NOSNAP}}, 0); + MockApplyImageStateRequest mock_apply_state_request; + EXPECT_CALL(mock_apply_state_request, send()) + .WillOnce(Invoke([this, &req=mock_apply_state_request, + &replayer=mock_replayer, &ctx=shutdown_ctx]() { + // inject a shutdown, to be pended due to STATE_REPLAYING + replayer.shut_down(&ctx); + m_threads->work_queue->queue(req.on_finish, -EINVAL); + })); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + // shutdown should be resumed + expect_unregister_update_watcher(mock_remote_image_ctx, 234, 0); + expect_unregister_update_watcher(mock_local_image_ctx, 123, 0); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + ASSERT_EQ(0, shutdown_ctx.wait()); +} + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc new file mode 100644 index 000000000..8a66cecae --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -0,0 +1,1195 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/BaseRequest.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/BootstrapRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenImageRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "tools/rbd_mirror/image_replayer/StateBuilder.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/image_sync/MockSyncPointHandler.h" +#include "test/rbd_mirror/mock/MockBaseRequest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +namespace rbd { +namespace mirror { + +class ProgressContext; + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +template<> +struct ImageSync<librbd::MockTestImageCtx> { + static ImageSync* s_instance; + Context *on_finish = nullptr; + + static ImageSync* create( + Threads<librbd::MockTestImageCtx>* threads, + librbd::MockTestImageCtx *local_image_ctx, + librbd::MockTestImageCtx *remote_image_ctx, + const std::string &local_mirror_uuid, + image_sync::SyncPointHandler* sync_point_handler, + InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher, + ProgressContext *progress_ctx, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + ImageSync() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~ImageSync() { + s_instance = nullptr; + } + + MOCK_METHOD0(get, void()); + MOCK_METHOD0(put, void()); + MOCK_METHOD0(send, void()); + MOCK_METHOD0(cancel, void()); +}; + +ImageSync<librbd::MockTestImageCtx>* + ImageSync<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { +}; + +namespace image_replayer { + +template<> +struct OpenImageRequest<librbd::MockTestImageCtx> { + static OpenImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static OpenImageRequest* create(librados::IoCtx &io_ctx, + librbd::MockTestImageCtx **image_ctx, + const std::string &image_id, + bool read_only, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + s_instance->construct(io_ctx, image_id); + return s_instance; + } + + OpenImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~OpenImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); + MOCK_METHOD0(send, void()); +}; + +template<> +struct OpenLocalImageRequest<librbd::MockTestImageCtx> { + static OpenLocalImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static OpenLocalImageRequest* create(librados::IoCtx &local_io_ctx, + librbd::MockTestImageCtx **local_image_ctx, + const std::string &local_image_id, + librbd::asio::ContextWQ *work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = local_image_ctx; + s_instance->on_finish = on_finish; + s_instance->construct(local_io_ctx, local_image_id); + return s_instance; + } + + OpenLocalImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~OpenLocalImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); + MOCK_METHOD0(send, void()); +}; + +template<> +struct PrepareLocalImageRequest<librbd::MockTestImageCtx> { + static PrepareLocalImageRequest* s_instance; + std::string *local_image_name = nullptr; + StateBuilder<librbd::MockTestImageCtx>** state_builder = nullptr; + Context *on_finish = nullptr; + + static PrepareLocalImageRequest* create(librados::IoCtx &, + const std::string &global_image_id, + std::string *local_image_name, + StateBuilder<librbd::MockTestImageCtx>** state_builder, + librbd::asio::ContextWQ *work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->local_image_name = local_image_name; + s_instance->state_builder = state_builder; + s_instance->on_finish = on_finish; + return s_instance; + } + + PrepareLocalImageRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> { + static PrepareRemoteImageRequest* s_instance; + StateBuilder<librbd::MockTestImageCtx>** state_builder = nullptr; + Context *on_finish = nullptr; + + static PrepareRemoteImageRequest* create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &, + librados::IoCtx &, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const RemotePoolMeta& remote_pool_meta, + ::journal::CacheManagerHandler *cache_manager_handler, + StateBuilder<librbd::MockTestImageCtx>** state_builder, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->state_builder = state_builder; + s_instance->on_finish = on_finish; + return s_instance; + } + + PrepareRemoteImageRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + static StateBuilder* s_instance; + + image_sync::MockSyncPointHandler mock_sync_point_handler; + MockBaseRequest mock_base_request; + + librbd::MockTestImageCtx* local_image_ctx = nullptr; + librbd::MockTestImageCtx* remote_image_ctx = nullptr; + std::string local_image_id; + std::string remote_mirror_uuid; + std::string remote_image_id; + + static StateBuilder* create(const std::string&) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + image_sync::MockSyncPointHandler* create_sync_point_handler() { + return &mock_sync_point_handler; + } + + StateBuilder() { + s_instance = this; + } + + MOCK_CONST_METHOD0(is_disconnected, bool()); + MOCK_CONST_METHOD0(is_local_primary, bool()); + MOCK_CONST_METHOD0(is_remote_primary, bool()); + MOCK_CONST_METHOD0(is_linked, bool()); + + MOCK_CONST_METHOD0(replay_requires_remote_image, bool()); + MOCK_METHOD1(close_remote_image, void(Context*)); + + MOCK_METHOD6(create_local_image_request, + BaseRequest*(Threads<librbd::MockTestImageCtx>*, + librados::IoCtx&, + const std::string&, + PoolMetaCache*, + ProgressContext*, + Context*)); + MOCK_METHOD5(create_prepare_replay_request, + BaseRequest*(const std::string&, + ProgressContext*, + bool*, bool*, Context*)); + + void destroy_sync_point_handler() { + } + void destroy() { + } +}; + +OpenImageRequest<librbd::MockTestImageCtx>* + OpenImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +OpenLocalImageRequest<librbd::MockTestImageCtx>* + OpenLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +PrepareLocalImageRequest<librbd::MockTestImageCtx>* + PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +PrepareRemoteImageRequest<librbd::MockTestImageCtx>* + PrepareRemoteImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +StateBuilder<librbd::MockTestImageCtx>* + StateBuilder<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/BootstrapRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +MATCHER_P(IsSameIoCtx, io_ctx, "") { + return &get_mock_io_ctx(arg) == &get_mock_io_ctx(*io_ctx); +} + +class TestMockImageReplayerBootstrapRequest : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest; + typedef ImageSync<librbd::MockTestImageCtx> MockImageSync; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef OpenImageRequest<librbd::MockTestImageCtx> MockOpenImageRequest; + typedef OpenLocalImageRequest<librbd::MockTestImageCtx> MockOpenLocalImageRequest; + typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest; + typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef std::list<cls::journal::Tag> Tags; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_send(MockPrepareLocalImageRequest &mock_request, + MockStateBuilder& mock_state_builder, + const std::string& local_image_id, + const std::string& local_image_name, int r) { + EXPECT_CALL(mock_request, send()) + .WillOnce(Invoke([&mock_request, &mock_state_builder, local_image_id, + local_image_name, r]() { + if (r == 0) { + *mock_request.state_builder = &mock_state_builder; + mock_state_builder.local_image_id = local_image_id; + *mock_request.local_image_name = local_image_name; + } + mock_request.on_finish->complete(r); + })); + } + + void expect_send(MockPrepareRemoteImageRequest& mock_request, + MockStateBuilder& mock_state_builder, + const std::string& remote_mirror_uuid, + const std::string& remote_image_id, + int r) { + EXPECT_CALL(mock_request, send()) + .WillOnce(Invoke([&mock_request, &mock_state_builder, remote_mirror_uuid, + remote_image_id, r]() { + if (r >= 0) { + *mock_request.state_builder = &mock_state_builder; + mock_state_builder.remote_image_id = remote_image_id; + } + + mock_state_builder.remote_mirror_uuid = remote_mirror_uuid; + mock_request.on_finish->complete(r); + })); + } + + void expect_is_local_primary(MockStateBuilder& mock_state_builder, + bool is_primary) { + EXPECT_CALL(mock_state_builder, is_local_primary()) + .WillOnce(Return(is_primary)); + } + + void expect_is_remote_primary(MockStateBuilder& mock_state_builder, + bool is_primary) { + EXPECT_CALL(mock_state_builder, is_remote_primary()) + .WillOnce(Return(is_primary)); + } + + void expect_is_linked(MockStateBuilder& mock_state_builder, bool is_linked) { + EXPECT_CALL(mock_state_builder, is_linked()) + .WillOnce(Return(is_linked)); + } + + void expect_is_disconnected(MockStateBuilder& mock_state_builder, + bool is_disconnected) { + EXPECT_CALL(mock_state_builder, is_disconnected()) + .WillOnce(Return(is_disconnected)); + } + + void expect_replay_requires_remote_image(MockStateBuilder& mock_state_builder, + bool requires_image) { + EXPECT_CALL(mock_state_builder, replay_requires_remote_image()) + .WillOnce(Return(requires_image)); + } + + void expect_open_image(MockOpenImageRequest &mock_open_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_open_image_request, + construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_image_request, send()) + .WillOnce(Invoke([this, &mock_open_image_request, &mock_image_ctx, r]() { + *mock_open_image_request.image_ctx = &mock_image_ctx; + m_threads->work_queue->queue(mock_open_image_request.on_finish, r); + })); + } + + void expect_open_local_image(MockOpenLocalImageRequest &mock_open_local_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx *mock_image_ctx, int r) { + EXPECT_CALL(mock_open_local_image_request, + construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_local_image_request, send()) + .WillOnce(Invoke([this, &mock_open_local_image_request, mock_image_ctx, r]() { + if (r >= 0) { + *mock_open_local_image_request.image_ctx = mock_image_ctx; + } + m_threads->work_queue->queue(mock_open_local_image_request.on_finish, + r); + })); + } + + void expect_close_remote_image( + MockStateBuilder& mock_state_builder, int r) { + EXPECT_CALL(mock_state_builder, close_remote_image(_)) + .WillOnce(Invoke([&mock_state_builder, r] + (Context* on_finish) { + mock_state_builder.remote_image_ctx = nullptr; + on_finish->complete(r); + })); + } + + void expect_create_local_image(MockStateBuilder& mock_state_builder, + const std::string& local_image_id, int r) { + EXPECT_CALL(mock_state_builder, + create_local_image_request(_, _, _, _, _, _)) + .WillOnce(WithArg<5>( + Invoke([&mock_state_builder, local_image_id, r](Context* ctx) { + if (r >= 0) { + mock_state_builder.local_image_id = local_image_id; + } + mock_state_builder.mock_base_request.on_finish = ctx; + return &mock_state_builder.mock_base_request; + }))); + EXPECT_CALL(mock_state_builder.mock_base_request, send()) + .WillOnce(Invoke([this, &mock_state_builder, r]() { + m_threads->work_queue->queue( + mock_state_builder.mock_base_request.on_finish, r); + })); + } + + void expect_prepare_replay(MockStateBuilder& mock_state_builder, + bool resync_requested, bool syncing, int r) { + EXPECT_CALL(mock_state_builder, + create_prepare_replay_request(_, _, _, _, _)) + .WillOnce(WithArgs<2, 3, 4>( + Invoke([&mock_state_builder, resync_requested, syncing, r] + (bool* resync, bool* sync, Context* ctx) { + if (r >= 0) { + *resync = resync_requested; + *sync = syncing; + } + mock_state_builder.mock_base_request.on_finish = ctx; + return &mock_state_builder.mock_base_request; + }))); + EXPECT_CALL(mock_state_builder.mock_base_request, send()) + .WillOnce(Invoke([this, &mock_state_builder, r]() { + m_threads->work_queue->queue( + mock_state_builder.mock_base_request.on_finish, r); + })); + } + + void expect_image_sync(MockImageSync &mock_image_sync, int r) { + EXPECT_CALL(mock_image_sync, get()); + EXPECT_CALL(mock_image_sync, send()) + .WillOnce(Invoke([this, &mock_image_sync, r]() { + m_threads->work_queue->queue(mock_image_sync.on_finish, r); + })); + EXPECT_CALL(mock_image_sync, put()); + } + + MockBootstrapRequest *create_request(MockThreads* mock_threads, + MockInstanceWatcher *mock_instance_watcher, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + Context *on_finish) { + return new MockBootstrapRequest(mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + mock_instance_watcher, + global_image_id, + local_mirror_uuid, + {"remote mirror uuid", + "remote mirror peer uuid"}, + nullptr, nullptr, nullptr, + &m_mock_state_builder, + &m_do_resync, on_finish); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx = nullptr; + + MockStateBuilder* m_mock_state_builder = nullptr; + bool m_do_resync = false; +}; + +TEST_F(TestMockImageReplayerBootstrapRequest, Success) { + InSequence seq; + + // prepare local image + MockStateBuilder mock_state_builder; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, false, 0); + expect_is_disconnected(mock_state_builder, false); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrepareRemoteImageNotPrimaryLocalDNE) { + InSequence seq; + + // prepare local image + MockStateBuilder mock_state_builder; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, -ENOENT); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, false); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(-EREMOTEIO, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrepareRemoteImageNotPrimaryLocalUnlinked) { + InSequence seq; + + // prepare local image + MockStateBuilder mock_state_builder; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, false); + expect_is_linked(mock_state_builder, false); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(-EREMOTEIO, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrepareRemoteImageNotPrimaryLocalLinked) { + InSequence seq; + + // prepare local image + MockStateBuilder mock_state_builder; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, false); + expect_is_linked(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, false, 0); + expect_is_disconnected(mock_state_builder, false); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, OpenLocalImageError) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, + -EINVAL); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, OpenLocalImageDNE) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, + -ENOENT); + + // create local image + expect_create_local_image(mock_state_builder, "local image id", 0); + + // re-open the local image + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + "local image id", &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, false, 0); + expect_is_disconnected(mock_state_builder, false); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, OpenLocalImagePrimary) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, + -EREMOTEIO); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(-EREMOTEIO, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, CreateLocalImageError) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, "", "", + -ENOENT); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // create local image + expect_create_local_image(mock_state_builder, "local image id", -EINVAL); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrepareReplayError) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, false, -EINVAL); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrepareReplayResyncRequested) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, true, false, 0); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(m_do_resync); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrepareReplaySyncing) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, true, 0); + expect_is_disconnected(mock_state_builder, false); + + // image sync + MockImageSync mock_image_sync; + expect_image_sync(mock_image_sync, 0); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrepareReplayDisconnected) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, false, 0); + expect_is_disconnected(mock_state_builder, true); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, ImageSyncError) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, true, 0); + expect_is_disconnected(mock_state_builder, false); + + // image sync + MockImageSync mock_image_sync; + expect_image_sync(mock_image_sync, -EINVAL); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, ImageSyncCanceled) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, true, 0); + expect_is_disconnected(mock_state_builder, false); + + // close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->cancel(); + request->send(); + ASSERT_EQ(-ECANCELED, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, CloseRemoteImageError) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, false, 0); + expect_is_disconnected(mock_state_builder, false); + + // attempt to close remote image + expect_replay_requires_remote_image(mock_state_builder, false); + expect_close_remote_image(mock_state_builder, -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, ReplayRequiresRemoteImage) { + InSequence seq; + + // prepare local image + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockStateBuilder mock_state_builder; + expect_send(mock_prepare_local_image_request, mock_state_builder, + m_local_image_ctx->id, m_local_image_ctx->name, 0); + + // prepare remote image + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + expect_send(mock_prepare_remote_image_request, mock_state_builder, + "remote mirror uuid", m_remote_image_ctx->id, 0); + expect_is_local_primary(mock_state_builder, false); + expect_is_remote_primary(mock_state_builder, true); + + // open the remote image + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // prepare replay + expect_prepare_replay(mock_state_builder, false, false, 0); + expect_is_disconnected(mock_state_builder, false); + + // remote image is left open + expect_replay_requires_remote_image(mock_state_builder, true); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, "global image id", + "local mirror uuid", &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc new file mode 100644 index 000000000..ed2cf8f96 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc @@ -0,0 +1,614 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenImageRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h" +#include "librbd/image/CreateRequest.h" +#include "librbd/image/CloneRequest.h" +#include "tools/rbd_mirror/Threads.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace image { + +template<> +struct CreateRequest<librbd::MockTestImageCtx> { + static CreateRequest *s_instance; + Context *on_finish = nullptr; + + static CreateRequest *create(const ConfigProxy& config, IoCtx &ioctx, + const std::string &imgname, + const std::string &imageid, uint64_t size, + const librbd::ImageOptions &image_options, + bool skip_mirror_enable, + cls::rbd::MirrorImageMode mode, + const std::string &non_primary_global_image_id, + const std::string &primary_mirror_uuid, + MockContextWQ *op_work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_FALSE(non_primary_global_image_id.empty()); + EXPECT_FALSE(primary_mirror_uuid.empty()); + EXPECT_FALSE(skip_mirror_enable); + s_instance->on_finish = on_finish; + s_instance->construct(ioctx); + return s_instance; + } + + CreateRequest() { + s_instance = this; + } + + ~CreateRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); + MOCK_METHOD1(construct, void(librados::IoCtx &ioctx)); +}; + +CreateRequest<librbd::MockTestImageCtx>* + CreateRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct CloneRequest<librbd::MockTestImageCtx> { + static CloneRequest *s_instance; + Context *on_finish = nullptr; + + static CloneRequest *create(ConfigProxy& config, IoCtx &p_ioctx, + const std::string &p_id, + const std::string &p_snap_name, + const cls::rbd::SnapshotNamespace& snap_ns, + uint64_t p_snap_id, + IoCtx &c_ioctx, const std::string &c_name, + const std::string &c_id, ImageOptions c_options, + cls::rbd::MirrorImageMode mode, + const std::string &non_primary_global_image_id, + const std::string &primary_mirror_uuid, + MockContextWQ *op_work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + s_instance->construct(); + return s_instance; + } + + CloneRequest() { + s_instance = this; + } + + ~CloneRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); + MOCK_METHOD0(construct, void()); +}; + +CloneRequest<librbd::MockTestImageCtx>* + CloneRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +namespace image_replayer { + +template<> +struct CloseImageRequest<librbd::MockTestImageCtx> { + static CloseImageRequest* s_instance; + Context *on_finish = nullptr; + + static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->construct(*image_ctx); + s_instance->on_finish = on_finish; + return s_instance; + } + + CloseImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~CloseImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD1(construct, void(librbd::MockTestImageCtx *image_ctx)); + MOCK_METHOD0(send, void()); +}; + +template<> +struct OpenImageRequest<librbd::MockTestImageCtx> { + static OpenImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static OpenImageRequest* create(librados::IoCtx &io_ctx, + librbd::MockTestImageCtx **image_ctx, + const std::string &image_id, + bool read_only, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + s_instance->construct(io_ctx, image_id); + return s_instance; + } + + OpenImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~OpenImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); + MOCK_METHOD0(send, void()); +}; + +CloseImageRequest<librbd::MockTestImageCtx>* + CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +OpenImageRequest<librbd::MockTestImageCtx>* + OpenImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.cc" +template class rbd::mirror::image_replayer::CreateImageRequest<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +MATCHER_P(IsSameIoCtx, io_ctx, "") { + return &get_mock_io_ctx(arg) == &get_mock_io_ctx(*io_ctx); +} + +class TestMockImageReplayerCreateImageRequest : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef librbd::image::CreateRequest<librbd::MockTestImageCtx> MockCreateRequest; + typedef librbd::image::CloneRequest<librbd::MockTestImageCtx> MockCloneRequest; + typedef CreateImageRequest<librbd::MockTestImageCtx> MockCreateImageRequest; + typedef OpenImageRequest<librbd::MockTestImageCtx> MockOpenImageRequest; + typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void snap_create(librbd::ImageCtx *image_ctx, const std::string &snap_name) { + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, image_ctx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + snap_name, 0, prog_ctx)); + ASSERT_EQ(0, image_ctx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + snap_name)); + ASSERT_EQ(0, image_ctx->state->refresh()); + } + + int clone_image(librbd::ImageCtx *parent_image_ctx, + const std::string &snap_name, const std::string &clone_name) { + snap_create(parent_image_ctx, snap_name); + + int order = 0; + return librbd::clone(m_remote_io_ctx, parent_image_ctx->name.c_str(), + snap_name.c_str(), m_remote_io_ctx, + clone_name.c_str(), parent_image_ctx->features, + &order, 0, 0); + } + + void expect_create_image(MockCreateRequest &mock_create_request, + librados::IoCtx &ioctx, int r) { + EXPECT_CALL(mock_create_request, construct(IsSameIoCtx(&ioctx))); + EXPECT_CALL(mock_create_request, send()) + .WillOnce(Invoke([this, &mock_create_request, r]() { + m_threads->work_queue->queue(mock_create_request.on_finish, r); + })); + } + + void expect_ioctx_create(librados::IoCtx &io_ctx) { + librados::MockTestMemIoCtxImpl &io_ctx_impl = get_mock_io_ctx(io_ctx); + EXPECT_CALL(*get_mock_io_ctx(io_ctx).get_mock_rados_client(), create_ioctx(_, _)) + .WillOnce(DoAll(GetReference(&io_ctx_impl), + Return(&get_mock_io_ctx(io_ctx)))); + } + + void expect_get_parent_global_image_id(librados::IoCtx &io_ctx, + const std::string &global_id, int r) { + cls::rbd::MirrorImage mirror_image; + mirror_image.global_image_id = global_id; + + bufferlist bl; + encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get"), + _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_mirror_image_get_image_id(librados::IoCtx &io_ctx, + const std::string &image_id, int r) { + bufferlist bl; + encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_get_image_id"), _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_open_image(MockOpenImageRequest &mock_open_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_open_image_request, construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_image_request, send()) + .WillOnce(Invoke([this, &mock_open_image_request, &mock_image_ctx, r]() { + *mock_open_image_request.image_ctx = &mock_image_ctx; + m_threads->work_queue->queue(mock_open_image_request.on_finish, r); + })); + } + + void expect_test_op_features(librbd::MockTestImageCtx& mock_image_ctx, + bool enabled) { + EXPECT_CALL(mock_image_ctx, + test_op_features(RBD_OPERATION_FEATURE_CLONE_CHILD)) + .WillOnce(Return(enabled)); + } + + void expect_clone_image(MockCloneRequest &mock_clone_request, + int r) { + EXPECT_CALL(mock_clone_request, construct()); + EXPECT_CALL(mock_clone_request, send()) + .WillOnce(Invoke([this, &mock_clone_request, r]() { + m_threads->work_queue->queue(mock_clone_request.on_finish, r); + })); + } + + void expect_close_image(MockCloseImageRequest &mock_close_image_request, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_close_image_request, construct(&mock_image_ctx)); + EXPECT_CALL(mock_close_image_request, send()) + .WillOnce(Invoke([this, &mock_close_image_request, r]() { + m_threads->work_queue->queue(mock_close_image_request.on_finish, r); + })); + } + + MockCreateImageRequest *create_request(MockThreads* mock_threads, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + librbd::MockTestImageCtx &mock_remote_image_ctx, + Context *on_finish) { + return new MockCreateImageRequest(mock_threads, m_local_io_ctx, + global_image_id, remote_mirror_uuid, + local_image_name, local_image_id, + &mock_remote_image_ctx, + &m_pool_meta_cache, + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + on_finish); + } + + PoolMetaCache m_pool_meta_cache{g_ceph_context}; + librbd::ImageCtx *m_remote_image_ctx; +}; + +TEST_F(TestMockImageReplayerCreateImageRequest, Create) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockCreateRequest mock_create_request; + + InSequence seq; + expect_create_image(mock_create_request, m_local_io_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_image_ctx, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CreateError) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockCreateRequest mock_create_request; + + InSequence seq; + expect_create_image(mock_create_request, m_local_io_ctx, -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_image_ctx, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", -ENOENT); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", -ENOENT); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockOpenImageRequest mock_open_image_request; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, + -ENOENT); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneParentImageSyncing) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + snap_create(m_remote_image_ctx, ".rbd-mirror.local parent uuid.1234"); + + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockOpenImageRequest mock_open_image_request; + MockCloseImageRequest mock_close_image_request; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneError) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockCloneRequest mock_clone_request; + MockOpenImageRequest mock_open_image_request; + MockCloseImageRequest mock_close_image_request; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_test_op_features(mock_remote_clone_image_ctx, false); + expect_clone_image(mock_clone_request, -EINVAL); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneRemoteParentCloseError) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + m_pool_meta_cache.set_local_pool_meta( + m_local_io_ctx.get_id(), {"local parent uuid"}); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockCloneRequest mock_clone_request; + MockOpenImageRequest mock_open_image_request; + MockCloseImageRequest mock_close_image_request; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_test_op_features(mock_remote_clone_image_ctx, false); + expect_clone_image(mock_clone_request, 0); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, + -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_GetMirrorImageIdRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_GetMirrorImageIdRequest.cc new file mode 100644 index 000000000..4a238d282 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_GetMirrorImageIdRequest.cc @@ -0,0 +1,107 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageReplayerGetMirrorImageIdRequest : public TestMockFixture { +public: + typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest; + + void expect_mirror_image_get_image_id(librados::IoCtx &io_ctx, + const std::string &image_id, int r) { + bufferlist bl; + encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_get_image_id"), _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + +}; + +TEST_F(TestMockImageReplayerGetMirrorImageIdRequest, Success) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "image id", 0); + + std::string image_id; + C_SaferCond ctx; + auto req = MockGetMirrorImageIdRequest::create(m_local_io_ctx, + "global image id", + &image_id, &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("image id"), image_id); +} + +TEST_F(TestMockImageReplayerGetMirrorImageIdRequest, MirrorImageIdDNE) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "", -ENOENT); + + std::string image_id; + C_SaferCond ctx; + auto req = MockGetMirrorImageIdRequest::create(m_local_io_ctx, + "global image id", + &image_id, &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerGetMirrorImageIdRequest, MirrorImageIdError) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "", -EINVAL); + + std::string image_id; + C_SaferCond ctx; + auto req = MockGetMirrorImageIdRequest::create(m_local_io_ctx, + "global image id", + &image_id, &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc new file mode 100644 index 000000000..54c3b24ef --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc @@ -0,0 +1,505 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/TypeTraits.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace mirror { + +template<> +struct GetInfoRequest<librbd::MockTestImageCtx> { + static GetInfoRequest* s_instance; + cls::rbd::MirrorImage *mirror_image; + PromotionState *promotion_state; + std::string *primary_mirror_uuid; + Context *on_finish = nullptr; + + static GetInfoRequest* create(librados::IoCtx& io_ctx, + librbd::asio::ContextWQ* context_wq, + const std::string& image_id, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->mirror_image = mirror_image; + s_instance->promotion_state = promotion_state; + s_instance->primary_mirror_uuid = primary_mirror_uuid; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetInfoRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~GetInfoRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +GetInfoRequest<librbd::MockTestImageCtx>* GetInfoRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct ImageDeleter<librbd::MockTestImageCtx> { + static ImageDeleter* s_instance; + + static void trash_move(librados::IoCtx& local_io_ctx, + const std::string& global_image_id, bool resync, + librbd::asio::ContextWQ* work_queue, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->trash_move(global_image_id, resync, on_finish); + } + + MOCK_METHOD3(trash_move, void(const std::string&, bool, Context*)); + + ImageDeleter() { + s_instance = this; + } +}; + +ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr; + +namespace image_replayer { + +template <> +struct GetMirrorImageIdRequest<librbd::MockTestImageCtx> { + static GetMirrorImageIdRequest* s_instance; + std::string* image_id = nullptr; + Context* on_finish = nullptr; + + static GetMirrorImageIdRequest* create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + std::string* image_id, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_id = image_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetMirrorImageIdRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + virtual ~StateBuilder() {} + + std::string local_image_id; + librbd::mirror::PromotionState local_promotion_state; +}; + +GetMirrorImageIdRequest<librbd::MockTestImageCtx>* GetMirrorImageIdRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +namespace journal { + +template<> +struct StateBuilder<librbd::MockTestImageCtx> + : public image_replayer::StateBuilder<librbd::MockTestImageCtx> { + static StateBuilder* s_instance; + + cls::rbd::MirrorImageMode mirror_image_mode = + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL; + + std::string local_primary_mirror_uuid; + + static StateBuilder* create(const std::string&) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + StateBuilder() { + s_instance = this; + } +}; + +StateBuilder<librbd::MockTestImageCtx>* StateBuilder<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace journal + +namespace snapshot { + +template<> +struct StateBuilder<librbd::MockTestImageCtx> + : public image_replayer::StateBuilder<librbd::MockTestImageCtx> { + static StateBuilder* s_instance; + + cls::rbd::MirrorImageMode mirror_image_mode = + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT; + + static StateBuilder* create(const std::string&) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + StateBuilder() { + s_instance = this; + } +}; + +StateBuilder<librbd::MockTestImageCtx>* StateBuilder<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageReplayerPrepareLocalImageRequest : public TestMockFixture { +public: + typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter; + typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest; + typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef journal::StateBuilder<librbd::MockTestImageCtx> MockJournalStateBuilder; + typedef snapshot::StateBuilder<librbd::MockTestImageCtx> MockSnapshotStateBuilder; + typedef librbd::mirror::GetInfoRequest<librbd::MockTestImageCtx> MockGetMirrorInfoRequest; + + void expect_get_mirror_image_id(MockGetMirrorImageIdRequest& mock_get_mirror_image_id_request, + const std::string& image_id, int r) { + EXPECT_CALL(mock_get_mirror_image_id_request, send()) + .WillOnce(Invoke([&mock_get_mirror_image_id_request, image_id, r]() { + *mock_get_mirror_image_id_request.image_id = image_id; + mock_get_mirror_image_id_request.on_finish->complete(r); + })); + } + + void expect_dir_get_name(librados::IoCtx &io_ctx, + const std::string &image_name, int r) { + bufferlist bl; + encode(image_name, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_DIRECTORY, _, StrEq("rbd"), StrEq("dir_get_name"), _, + _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_get_mirror_info( + MockGetMirrorInfoRequest &mock_get_mirror_info_request, + const cls::rbd::MirrorImage &mirror_image, + librbd::mirror::PromotionState promotion_state, + const std::string& primary_mirror_uuid, int r) { + EXPECT_CALL(mock_get_mirror_info_request, send()) + .WillOnce(Invoke([this, &mock_get_mirror_info_request, mirror_image, + promotion_state, primary_mirror_uuid, r]() { + *mock_get_mirror_info_request.mirror_image = mirror_image; + *mock_get_mirror_info_request.promotion_state = promotion_state; + *mock_get_mirror_info_request.primary_mirror_uuid = + primary_mirror_uuid; + m_threads->work_queue->queue( + mock_get_mirror_info_request.on_finish, r); + })); + } + + void expect_trash_move(MockImageDeleter& mock_image_deleter, + const std::string& global_image_id, + bool ignore_orphan, int r) { + EXPECT_CALL(mock_image_deleter, + trash_move(global_image_id, ignore_orphan, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + +}; + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, SuccessJournal) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(mock_state_builder != nullptr); + ASSERT_EQ(std::string("local image name"), local_image_name); + ASSERT_EQ(std::string("local image id"), + mock_journal_state_builder.local_image_id); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + mock_journal_state_builder.mirror_image_mode); + ASSERT_EQ(librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + mock_journal_state_builder.local_promotion_state); + ASSERT_EQ(std::string("remote mirror uuid"), + mock_journal_state_builder.local_primary_mirror_uuid); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, SuccessSnapshot) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + MockSnapshotStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(mock_state_builder != nullptr); + ASSERT_EQ(std::string("local image name"), local_image_name); + ASSERT_EQ(std::string("local image id"), + mock_journal_state_builder.local_image_id); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + mock_journal_state_builder.mirror_image_mode); + ASSERT_EQ(librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + mock_journal_state_builder.local_promotion_state); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageIdError) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "", -EINVAL); + + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, DirGetNameDNE) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "", -ENOENT); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, DirGetNameError) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "", -EPERM); + + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageInfoError) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", -EINVAL); + + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, ImageCreating) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_CREATING}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + MockImageDeleter mock_image_deleter; + expect_trash_move(mock_image_deleter, "global image id", false, 0); + + MockSnapshotStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_TRUE(mock_state_builder == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, ImageDisabling) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + MockSnapshotStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ERESTART, ctx.wait()); + ASSERT_TRUE(mock_state_builder == nullptr); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc new file mode 100644 index 000000000..e5b473c0f --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc @@ -0,0 +1,811 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/TypeTraits.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "tools/rbd_mirror/image_replayer/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<MockTestImageCtx> { + typedef ::journal::MockJournalerProxy Journaler; +}; + +} // namespace journal + +namespace mirror { + +template<> +struct GetInfoRequest<librbd::MockTestImageCtx> { + static GetInfoRequest* s_instance; + cls::rbd::MirrorImage *mirror_image; + PromotionState *promotion_state; + std::string *primary_mirror_uuid; + Context *on_finish = nullptr; + + static GetInfoRequest* create(librados::IoCtx& io_ctx, + librbd::asio::ContextWQ* context_wq, + const std::string& image_id, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->mirror_image = mirror_image; + s_instance->promotion_state = promotion_state; + s_instance->primary_mirror_uuid = primary_mirror_uuid; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetInfoRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~GetInfoRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +GetInfoRequest<librbd::MockTestImageCtx>* GetInfoRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +namespace image_replayer { + +template <> +struct GetMirrorImageIdRequest<librbd::MockTestImageCtx> { + static GetMirrorImageIdRequest* s_instance; + std::string* image_id = nullptr; + Context* on_finish = nullptr; + + static GetMirrorImageIdRequest* create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + std::string* image_id, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_id = image_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetMirrorImageIdRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct StateBuilder<librbd::MockTestImageCtx> { + std::string local_image_id; + librbd::mirror::PromotionState local_promotion_state = + librbd::mirror::PROMOTION_STATE_NON_PRIMARY; + std::string remote_image_id; + std::string remote_mirror_uuid; + librbd::mirror::PromotionState remote_promotion_state; + + virtual ~StateBuilder() {} + + MOCK_CONST_METHOD0(get_mirror_image_mode, cls::rbd::MirrorImageMode()); +}; + +GetMirrorImageIdRequest<librbd::MockTestImageCtx>* GetMirrorImageIdRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +namespace journal { + +template<> +struct StateBuilder<librbd::MockTestImageCtx> + : public image_replayer::StateBuilder<librbd::MockTestImageCtx> { + static StateBuilder* s_instance; + + cls::rbd::MirrorImageMode mirror_image_mode = + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL; + + ::journal::MockJournalerProxy* remote_journaler = nullptr; + cls::journal::ClientState remote_client_state; + librbd::journal::MirrorPeerClientMeta remote_client_meta; + + static StateBuilder* create(const std::string&) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + StateBuilder() { + s_instance = this; + } +}; + +StateBuilder<librbd::MockTestImageCtx>* StateBuilder<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace journal + +namespace snapshot { + +template<> +struct StateBuilder<librbd::MockTestImageCtx> + : public image_replayer::StateBuilder<librbd::MockTestImageCtx> { + static StateBuilder* s_instance; + + cls::rbd::MirrorImageMode mirror_image_mode = + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT; + + std::string remote_mirror_peer_uuid; + + static StateBuilder* create(const std::string&) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + StateBuilder() { + s_instance = this; + } +}; + +StateBuilder<librbd::MockTestImageCtx>* StateBuilder<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageReplayerPrepareRemoteImageRequest : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest; + typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest; + typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef journal::StateBuilder<librbd::MockTestImageCtx> MockJournalStateBuilder; + typedef snapshot::StateBuilder<librbd::MockTestImageCtx> MockSnapshotStateBuilder; + typedef librbd::mirror::GetInfoRequest<librbd::MockTestImageCtx> MockGetMirrorInfoRequest; + + void expect_get_mirror_image_mode(MockStateBuilder& mock_state_builder, + cls::rbd::MirrorImageMode mirror_image_mode) { + EXPECT_CALL(mock_state_builder, get_mirror_image_mode()) + .WillOnce(Return(mirror_image_mode)); + } + + void expect_get_mirror_image_id(MockGetMirrorImageIdRequest& mock_get_mirror_image_id_request, + const std::string& image_id, int r) { + EXPECT_CALL(mock_get_mirror_image_id_request, send()) + .WillOnce(Invoke([&mock_get_mirror_image_id_request, image_id, r]() { + *mock_get_mirror_image_id_request.image_id = image_id; + mock_get_mirror_image_id_request.on_finish->complete(r); + })); + } + + void expect_get_mirror_info( + MockGetMirrorInfoRequest &mock_get_mirror_info_request, + const cls::rbd::MirrorImage &mirror_image, + librbd::mirror::PromotionState promotion_state, + const std::string& primary_mirror_uuid, int r) { + EXPECT_CALL(mock_get_mirror_info_request, send()) + .WillOnce(Invoke([this, &mock_get_mirror_info_request, mirror_image, + promotion_state, primary_mirror_uuid, r]() { + *mock_get_mirror_info_request.mirror_image = mirror_image; + *mock_get_mirror_info_request.promotion_state = promotion_state; + *mock_get_mirror_info_request.primary_mirror_uuid = + primary_mirror_uuid; + m_threads->work_queue->queue( + mock_get_mirror_info_request.on_finish, r); + })); + } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_register_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } +}; + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessJournal) { + ::journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + cls::journal::Client client; + client.state = cls::journal::CLIENT_STATE_DISCONNECTED; + encode(client_data, client.data); + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, 0); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(mock_state_builder != nullptr); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + mock_journal_state_builder.mirror_image_mode); + ASSERT_EQ(std::string("remote mirror uuid"), + mock_journal_state_builder.remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), + mock_journal_state_builder.remote_image_id); + ASSERT_EQ(librbd::mirror::PROMOTION_STATE_PRIMARY, + mock_journal_state_builder.remote_promotion_state); + ASSERT_TRUE(mock_journal_state_builder.remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_DISCONNECTED, + mock_journal_state_builder.remote_client_state); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessSnapshot) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + MockSnapshotStateBuilder mock_snapshot_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", + "remote mirror peer uuid"}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(mock_state_builder != nullptr); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + mock_snapshot_state_builder.mirror_image_mode); + ASSERT_EQ(std::string("remote mirror uuid"), + mock_snapshot_state_builder.remote_mirror_uuid); + ASSERT_EQ(std::string("remote mirror peer uuid"), + mock_snapshot_state_builder.remote_mirror_peer_uuid); + ASSERT_EQ(std::string("remote image id"), + mock_snapshot_state_builder.remote_image_id); + ASSERT_EQ(librbd::mirror::PROMOTION_STATE_PRIMARY, + mock_snapshot_state_builder.remote_promotion_state); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) { + ::journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + MockJournalStateBuilder mock_journal_state_builder; + expect_get_mirror_image_mode(mock_journal_state_builder, + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, 0); + + mock_journal_state_builder.local_image_id = "local image id"; + MockStateBuilder* mock_state_builder = &mock_journal_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(mock_state_builder != nullptr); + ASSERT_EQ(std::string("remote image id"), + mock_journal_state_builder.remote_image_id); + ASSERT_EQ(librbd::mirror::PROMOTION_STATE_PRIMARY, + mock_journal_state_builder.remote_promotion_state); + ASSERT_TRUE(mock_journal_state_builder.remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_CONNECTED, + mock_journal_state_builder.remote_client_state); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetMirrorImageIdError) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "", -EINVAL); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = &mock_journal_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_TRUE(mock_journal_state_builder.remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetMirrorInfoError) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", -EINVAL); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_TRUE(mock_state_builder == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) { + ::journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -EINVAL); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_TRUE(mock_state_builder == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) { + ::journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + MockJournalStateBuilder mock_journal_state_builder; + expect_get_mirror_image_mode(mock_journal_state_builder, + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, -EINVAL); + + mock_journal_state_builder.local_image_id = "local image id"; + MockStateBuilder* mock_state_builder = &mock_journal_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdDNEJournal) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "", -ENOENT); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = &mock_journal_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + mock_journal_state_builder.mirror_image_mode); + ASSERT_EQ("remote mirror uuid", + mock_journal_state_builder.remote_mirror_uuid); + ASSERT_EQ("", mock_journal_state_builder.remote_image_id); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdDNESnapshot) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "", -ENOENT); + + MockSnapshotStateBuilder mock_snapshot_state_builder; + MockStateBuilder* mock_state_builder = &mock_snapshot_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", + "remote mirror peer uuid"}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + mock_snapshot_state_builder.mirror_image_mode); + ASSERT_EQ("remote mirror uuid", + mock_snapshot_state_builder.remote_mirror_uuid); + ASSERT_EQ("remote mirror peer uuid", + mock_snapshot_state_builder.remote_mirror_peer_uuid); + ASSERT_EQ("", mock_snapshot_state_builder.remote_image_id); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorInfoDNEJournal) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", -ENOENT); + + MockJournalStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = &mock_journal_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + mock_journal_state_builder.mirror_image_mode); + ASSERT_EQ("remote mirror uuid", + mock_journal_state_builder.remote_mirror_uuid); + ASSERT_EQ("", mock_journal_state_builder.remote_image_id); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorInfoDNESnapshot) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_ENABLED}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", -ENOENT); + + MockSnapshotStateBuilder mock_snapshot_state_builder; + MockStateBuilder* mock_state_builder = &mock_snapshot_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", + "remote mirror peer uuid"}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + mock_snapshot_state_builder.mirror_image_mode); + ASSERT_EQ("remote mirror uuid", + mock_snapshot_state_builder.remote_mirror_uuid); + ASSERT_EQ("remote mirror peer uuid", + mock_snapshot_state_builder.remote_mirror_peer_uuid); + ASSERT_EQ("", mock_snapshot_state_builder.remote_image_id); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorInfoDisablingJournal) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + MockJournalStateBuilder mock_journal_state_builder; + expect_get_mirror_image_mode(mock_journal_state_builder, + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL); + MockStateBuilder* mock_state_builder = &mock_journal_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", ""}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, + mock_journal_state_builder.mirror_image_mode); + ASSERT_EQ("remote mirror uuid", + mock_journal_state_builder.remote_mirror_uuid); + ASSERT_EQ("", mock_journal_state_builder.remote_image_id); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorInfoDisablingSnapshot) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, + librbd::mirror::PROMOTION_STATE_PRIMARY, + "remote mirror uuid", 0); + + MockSnapshotStateBuilder mock_snapshot_state_builder; + expect_get_mirror_image_mode(mock_snapshot_state_builder, + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT); + MockStateBuilder* mock_state_builder = &mock_snapshot_state_builder; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_local_io_ctx, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + {"remote mirror uuid", + "remote mirror peer uuid"}, + nullptr, + &mock_state_builder, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + mock_snapshot_state_builder.mirror_image_mode); + ASSERT_EQ("remote mirror uuid", + mock_snapshot_state_builder.remote_mirror_uuid); + ASSERT_EQ("remote mirror peer uuid", + mock_snapshot_state_builder.remote_mirror_peer_uuid); + ASSERT_EQ("", mock_snapshot_state_builder.remote_image_id); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_sync/test_mock_SyncPointCreateRequest.cc b/src/test/rbd_mirror/image_sync/test_mock_SyncPointCreateRequest.cc new file mode 100644 index 000000000..9a4d920f8 --- /dev/null +++ b/src/test/rbd_mirror/image_sync/test_mock_SyncPointCreateRequest.cc @@ -0,0 +1,195 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/image_sync/MockSyncPointHandler.h" +#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_sync { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::WithArg; + +class TestMockImageSyncSyncPointCreateRequest : public TestMockFixture { +public: + typedef SyncPointCreateRequest<librbd::MockTestImageCtx> MockSyncPointCreateRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void expect_get_snap_seqs(MockSyncPointHandler& mock_sync_point_handler) { + EXPECT_CALL(mock_sync_point_handler, get_snap_seqs()) + .WillRepeatedly(Return(librbd::SnapSeqs{})); + } + + void expect_get_sync_points(MockSyncPointHandler& mock_sync_point_handler) { + EXPECT_CALL(mock_sync_point_handler, get_sync_points()) + .WillRepeatedly(Invoke([this]() { + return m_sync_points; + })); + } + + void expect_update_sync_points(MockSyncPointHandler& mock_sync_point_handler, + int r) { + EXPECT_CALL(mock_sync_point_handler, update_sync_points(_, _, false, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([this, r](const SyncPoints& sync_points) { + if (r >= 0) { + m_sync_points = sync_points; + } + })), + WithArg<3>(CompleteContext(r)))); + } + + void expect_image_refresh(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.state, refresh(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_snap_create(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.operations, snap_create(_, _, _, _, _)) + .WillOnce(WithArg<4>(CompleteContext(r))); + } + + MockSyncPointCreateRequest *create_request(librbd::MockTestImageCtx &mock_remote_image_ctx, + MockSyncPointHandler& mock_sync_point_handler, + Context *ctx) { + return new MockSyncPointCreateRequest(&mock_remote_image_ctx, "uuid", + &mock_sync_point_handler, ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + SyncPoints m_sync_points; +}; + +TEST_F(TestMockImageSyncSyncPointCreateRequest, Success) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_update_sync_points(mock_sync_point_handler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(1U, m_sync_points.size()); +} + +TEST_F(TestMockImageSyncSyncPointCreateRequest, ResyncSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "start snap", + "", boost::none); + auto sync_point = m_sync_points.front(); + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_update_sync_points(mock_sync_point_handler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(2U, m_sync_points.size()); + ASSERT_EQ(sync_point, m_sync_points.front()); + ASSERT_EQ("start snap", m_sync_points.back().from_snap_name); +} + +TEST_F(TestMockImageSyncSyncPointCreateRequest, SnapshotExists) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_update_sync_points(mock_sync_point_handler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, -EEXIST); + expect_update_sync_points(mock_sync_point_handler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(1U, m_sync_points.size()); +} + +TEST_F(TestMockImageSyncSyncPointCreateRequest, ClientUpdateError) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_update_sync_points(mock_sync_point_handler, -EINVAL); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); + + ASSERT_TRUE(m_sync_points.empty()); +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_sync/test_mock_SyncPointPruneRequest.cc b/src/test/rbd_mirror/image_sync/test_mock_SyncPointPruneRequest.cc new file mode 100644 index 000000000..bd13f3cd0 --- /dev/null +++ b/src/test/rbd_mirror/image_sync/test_mock_SyncPointPruneRequest.cc @@ -0,0 +1,347 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/image_sync/MockSyncPointHandler.h" +#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc" +template class rbd::mirror::image_sync::SyncPointPruneRequest<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace image_sync { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageSyncSyncPointPruneRequest : public TestMockFixture { +public: + typedef SyncPointPruneRequest<librbd::MockTestImageCtx> MockSyncPointPruneRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void expect_get_snap_seqs(MockSyncPointHandler& mock_sync_point_handler) { + EXPECT_CALL(mock_sync_point_handler, get_snap_seqs()) + .WillRepeatedly(Return(librbd::SnapSeqs{})); + } + + void expect_get_sync_points(MockSyncPointHandler& mock_sync_point_handler) { + EXPECT_CALL(mock_sync_point_handler, get_sync_points()) + .WillRepeatedly(Invoke([this]() { + return m_sync_points; + })); + } + + void expect_update_sync_points(MockSyncPointHandler& mock_sync_point_handler, + bool complete, int r) { + EXPECT_CALL(mock_sync_point_handler, update_sync_points(_, _, complete, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([this, r](const SyncPoints& sync_points) { + if (r >= 0) { + m_sync_points = sync_points; + } + })), + WithArg<3>(CompleteContext(r)))); + } + + void expect_get_snap_id(librbd::MockTestImageCtx &mock_remote_image_ctx, + const std::string &snap_name, uint64_t snap_id) { + EXPECT_CALL(mock_remote_image_ctx, get_snap_id(_, StrEq(snap_name))) + .WillOnce(Return(snap_id)); + } + + void expect_image_refresh(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.state, refresh(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_snap_remove(librbd::MockTestImageCtx &mock_remote_image_ctx, + const std::string &snap_name, int r) { + EXPECT_CALL(*mock_remote_image_ctx.operations, snap_remove(_, StrEq(snap_name), _)) + .WillOnce(WithArg<2>(CompleteContext(r))); + } + + MockSyncPointPruneRequest *create_request(librbd::MockTestImageCtx &mock_remote_image_ctx, + MockSyncPointHandler& mock_sync_point_handler, + bool sync_complete, Context *ctx) { + return new MockSyncPointPruneRequest(&mock_remote_image_ctx, sync_complete, + &mock_sync_point_handler, ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + SyncPoints m_sync_points; +}; + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncInProgressSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap1", + "", boost::none); + auto sync_points = m_sync_points; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap1", 123); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, false, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(sync_points, m_sync_points); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, RestartedSyncInProgressSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap2", + "snap1", boost::none); + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap1", "", + boost::none); + auto sync_points = m_sync_points; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap1", 123); + expect_snap_remove(mock_remote_image_ctx, "snap2", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, false, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + sync_points.pop_back(); + ASSERT_EQ(sync_points, m_sync_points); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncInProgressMissingSnapSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap2", + "snap1", boost::none); + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap1", "", + boost::none); + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap1", CEPH_NOSNAP); + expect_snap_remove(mock_remote_image_ctx, "snap2", 0); + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, false, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(SyncPoints{}, m_sync_points); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncInProgressUnexpectedFromSnapSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap2", + "snap1", boost::none); + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap2", 124); + expect_snap_remove(mock_remote_image_ctx, "snap2", 0); + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, false, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(SyncPoints(), m_sync_points); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncCompleteSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap1", + "", boost::none); + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, true, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(m_sync_points.empty()); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, RestartedSyncCompleteSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap2", + "snap1", boost::none); + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap1", + "", boost::none); + auto sync_points = m_sync_points; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, true, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + sync_points.pop_front(); + ASSERT_EQ(sync_points, m_sync_points); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, RestartedCatchUpSyncCompleteSuccess) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap3", + "snap2", boost::none); + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap2", + "snap1", boost::none); + auto sync_points = m_sync_points; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, true, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + sync_points.pop_front(); + ASSERT_EQ(sync_points, m_sync_points); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SnapshotDNE) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap1", + "", boost::none); + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_snap_remove(mock_remote_image_ctx, "snap1", -ENOENT); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, true, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(m_sync_points.empty()); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, ClientUpdateError) { + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap2", + "snap1", boost::none); + m_sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), "snap1", + "", boost::none); + auto sync_points = m_sync_points; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_sync_points(mock_sync_point_handler, true, -EINVAL); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_sync_point_handler, + true, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); + + ASSERT_EQ(sync_points, m_sync_points); +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/mock/MockBaseRequest.h b/src/test/rbd_mirror/mock/MockBaseRequest.h new file mode 100644 index 000000000..c85eab434 --- /dev/null +++ b/src/test/rbd_mirror/mock/MockBaseRequest.h @@ -0,0 +1,26 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_MOCK_BASE_REQUEST_H +#define CEPH_MOCK_BASE_REQUEST_H + +#include "tools/rbd_mirror/BaseRequest.h" +#include <gmock/gmock.h> + +struct Context; + +namespace rbd { +namespace mirror { + +struct MockBaseRequest : public BaseRequest { + MockBaseRequest() : BaseRequest(nullptr) {} + + Context* on_finish = nullptr; + + MOCK_METHOD0(send, void()); +}; + +} // namespace mirror +} // namepace rbd + +#endif // CEPH_MOCK_BASE_REQUEST_H diff --git a/src/test/rbd_mirror/mock/MockContextWQ.h b/src/test/rbd_mirror/mock/MockContextWQ.h new file mode 100644 index 000000000..1c0ee88f5 --- /dev/null +++ b/src/test/rbd_mirror/mock/MockContextWQ.h @@ -0,0 +1,18 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_MOCK_CONTEXT_WQ_H +#define CEPH_MOCK_CONTEXT_WQ_H + +#include <gmock/gmock.h> + +struct Context; + +struct MockContextWQ { + void queue(Context *ctx) { + queue(ctx, 0); + } + MOCK_METHOD2(queue, void(Context *, int)); +}; + +#endif // CEPH_MOCK_CONTEXT_WQ_H diff --git a/src/test/rbd_mirror/mock/MockSafeTimer.h b/src/test/rbd_mirror/mock/MockSafeTimer.h new file mode 100644 index 000000000..32d58471d --- /dev/null +++ b/src/test/rbd_mirror/mock/MockSafeTimer.h @@ -0,0 +1,16 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_MOCK_SAFE_TIMER_H +#define CEPH_MOCK_SAFE_TIMER_H + +#include <gmock/gmock.h> + +struct Context; + +struct MockSafeTimer { + MOCK_METHOD2(add_event_after, Context*(double, Context*)); + MOCK_METHOD1(cancel_event, bool(Context *)); +}; + +#endif // CEPH_MOCK_SAFE_TIMER_H diff --git a/src/test/rbd_mirror/mock/image_sync/MockSyncPointHandler.h b/src/test/rbd_mirror/mock/image_sync/MockSyncPointHandler.h new file mode 100644 index 000000000..b6263cbdf --- /dev/null +++ b/src/test/rbd_mirror/mock/image_sync/MockSyncPointHandler.h @@ -0,0 +1,29 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_MOCK_IMAGE_SYNC_SYNC_POINT_HANDLER_H +#define CEPH_MOCK_IMAGE_SYNC_SYNC_POINT_HANDLER_H + +#include "tools/rbd_mirror/image_sync/Types.h" +#include <gmock/gmock.h> + +struct Context; + +namespace rbd { +namespace mirror { +namespace image_sync { + +struct MockSyncPointHandler : public SyncPointHandler{ + MOCK_CONST_METHOD0(get_sync_points, SyncPoints()); + MOCK_CONST_METHOD0(get_snap_seqs, librbd::SnapSeqs()); + + MOCK_METHOD4(update_sync_points, void(const librbd::SnapSeqs&, + const SyncPoints&, + bool, Context*)); +}; + +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +#endif // CEPH_MOCK_IMAGE_SYNC_SYNC_POINT_HANDLER_H diff --git a/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc new file mode 100644 index 000000000..3347a6cd4 --- /dev/null +++ b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc @@ -0,0 +1,117 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h" +#include "include/stringify.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.cc" +template class rbd::mirror::pool_watcher::RefreshImagesRequest<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace pool_watcher { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockPoolWatcherRefreshImagesRequest : public TestMockFixture { +public: + typedef RefreshImagesRequest<librbd::MockTestImageCtx> MockRefreshImagesRequest; + + void expect_mirror_image_list(librados::IoCtx &io_ctx, + const std::map<std::string, std::string> &ids, + int r) { + bufferlist bl; + encode(ids, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_list"), + _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + +}; + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, Success) { + InSequence seq; + expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ImageIds expected_image_ids = {{"global id", "local id"}}; + ASSERT_EQ(expected_image_ids, image_ids); +} + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, LargeDirectory) { + InSequence seq; + std::map<std::string, std::string> mirror_list; + ImageIds expected_image_ids; + for (uint32_t idx = 1; idx <= 1024; ++idx) { + mirror_list.insert(std::make_pair("local id " + stringify(idx), + "global id " + stringify(idx))); + expected_image_ids.insert({{"global id " + stringify(idx), + "local id " + stringify(idx)}}); + } + + expect_mirror_image_list(m_remote_io_ctx, mirror_list, 0); + expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(0, ctx.wait()); + + expected_image_ids.insert({"global id", "local id"}); + ASSERT_EQ(expected_image_ids, image_ids); +} + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, MirrorImageListError) { + InSequence seq; + expect_mirror_image_list(m_remote_io_ctx, {}, -EINVAL); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace pool_watcher +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/random_write.cc b/src/test/rbd_mirror/random_write.cc new file mode 100644 index 000000000..6cb544c45 --- /dev/null +++ b/src/test/rbd_mirror/random_write.cc @@ -0,0 +1,210 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/ceph_argparse.h" +#include "common/config.h" +#include "common/debug.h" +#include "common/errno.h" +#include "common/Cond.h" +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "global/global_init.h" +#include <string> +#include <vector> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "random-write: " + +namespace { + +const uint32_t NUM_THREADS = 8; +const uint32_t MAX_IO_SIZE = 24576; +const uint32_t MIN_IO_SIZE = 4; + +void usage() { + std::cout << "usage: ceph_test_rbd_mirror_random_write [options...] \\" << std::endl; + std::cout << " <pool> <image>" << std::endl; + std::cout << std::endl; + std::cout << " pool image pool" << std::endl; + std::cout << " image image to write" << std::endl; + std::cout << std::endl; + std::cout << "options:\n"; + std::cout << " -m monaddress[:port] connect to specified monitor\n"; + std::cout << " --keyring=<path> path to keyring for local cluster\n"; + std::cout << " --log-file=<logfile> file to log debug output\n"; + std::cout << " --debug-rbd-mirror=<log-level>/<memory-level> set rbd-mirror debug level\n"; + generic_server_usage(); +} + +void rbd_bencher_completion(void *c, void *pc); + +struct rbd_bencher { + librbd::Image *image; + ceph::mutex lock = ceph::make_mutex("rbd_bencher::lock"); + ceph::condition_variable cond; + int in_flight; + + explicit rbd_bencher(librbd::Image *i) + : image(i), + in_flight(0) { + } + + bool start_write(int max, uint64_t off, uint64_t len, bufferlist& bl, + int op_flags) { + { + std::lock_guard l{lock}; + if (in_flight >= max) + return false; + in_flight++; + } + librbd::RBD::AioCompletion *c = + new librbd::RBD::AioCompletion((void *)this, rbd_bencher_completion); + image->aio_write2(off, len, bl, c, op_flags); + //cout << "start " << c << " at " << off << "~" << len << std::endl; + return true; + } + + void wait_for(int max) { + std::unique_lock l{lock}; + while (in_flight > max) { + cond.wait_for(l, 200ms); + } + } + +}; + +void rbd_bencher_completion(void *vc, void *pc) { + librbd::RBD::AioCompletion *c = (librbd::RBD::AioCompletion *)vc; + rbd_bencher *b = static_cast<rbd_bencher *>(pc); + //cout << "complete " << c << std::endl; + int ret = c->get_return_value(); + if (ret != 0) { + cout << "write error: " << cpp_strerror(ret) << std::endl; + exit(ret < 0 ? -ret : ret); + } + b->lock.lock(); + b->in_flight--; + b->cond.notify_all(); + b->lock.unlock(); + c->release(); +} + +void write_image(librbd::Image &image) { + srand(time(NULL) % (unsigned long) -1); + + uint64_t max_io_bytes = MAX_IO_SIZE * 1024; + bufferptr bp(max_io_bytes); + memset(bp.c_str(), rand() & 0xff, bp.length()); + bufferlist bl; + bl.push_back(bp); + + uint64_t size = 0; + image.size(&size); + ceph_assert(size != 0); + + vector<uint64_t> thread_offset; + uint64_t i; + uint64_t start_pos; + + // disturb all thread's offset, used by seq write + for (i = 0; i < NUM_THREADS; i++) { + start_pos = (rand() % (size / max_io_bytes)) * max_io_bytes; + thread_offset.push_back(start_pos); + } + + uint64_t total_ios = 0; + uint64_t total_bytes = 0; + rbd_bencher b(&image); + while (true) { + b.wait_for(NUM_THREADS - 1); + for (uint32_t i = 0; i < NUM_THREADS; ++i) { + // mostly small writes with a small chance of large writes + uint32_t io_modulo = MIN_IO_SIZE + 1; + if (rand() % 30 == 0) { + io_modulo += MAX_IO_SIZE; + } + + uint32_t io_size = (((rand() % io_modulo) + MIN_IO_SIZE) * 1024); + thread_offset[i] = (rand() % (size / io_size)) * io_size; + if (!b.start_write(NUM_THREADS, thread_offset[i], io_size, bl, + LIBRADOS_OP_FLAG_FADVISE_RANDOM)) { + break; + } + ++i; + + ++total_ios; + total_bytes += io_size; + if (total_ios % 100 == 0) { + std::cout << total_ios << " IOs, " << total_bytes << " bytes" + << std::endl; + } + } + } + b.wait_for(0); +} + +} // anonymous namespace + +int main(int argc, const char **argv) +{ + std::vector<const char*> args; + argv_to_vec(argc, argv, args); + if (args.empty()) { + cerr << argv[0] << ": -h or --help for usage" << std::endl; + exit(1); + } + if (ceph_argparse_need_usage(args)) { + usage(); + exit(0); + } + + auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_MON_CONFIG); + + if (args.size() < 2) { + usage(); + return EXIT_FAILURE; + } + + std::string pool_name = args[0]; + std::string image_name = args[1]; + + common_init_finish(g_ceph_context); + + dout(5) << "connecting to cluster" << dendl; + librados::Rados rados; + librados::IoCtx io_ctx; + librbd::RBD rbd; + librbd::Image image; + int r = rados.init_with_context(g_ceph_context); + if (r < 0) { + derr << "could not initialize RADOS handle" << dendl; + return EXIT_FAILURE; + } + + r = rados.connect(); + if (r < 0) { + derr << "error connecting to local cluster" << dendl; + return EXIT_FAILURE; + } + + r = rados.ioctx_create(pool_name.c_str(), io_ctx); + if (r < 0) { + derr << "error finding local pool " << pool_name << ": " + << cpp_strerror(r) << dendl; + return EXIT_FAILURE; + } + + r = rbd.open(io_ctx, image, image_name.c_str()); + if (r < 0) { + derr << "error opening image " << image_name << ": " + << cpp_strerror(r) << dendl; + return EXIT_FAILURE; + } + + write_image(image); + return EXIT_SUCCESS; +} diff --git a/src/test/rbd_mirror/test_ClusterWatcher.cc b/src/test/rbd_mirror/test_ClusterWatcher.cc new file mode 100644 index 000000000..03fd3b142 --- /dev/null +++ b/src/test/rbd_mirror/test_ClusterWatcher.cc @@ -0,0 +1,265 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#include "include/rados/librados.hpp" +#include "common/Cond.h" +#include "common/errno.h" +#include "common/ceph_mutex.h" +#include "librbd/internal.h" +#include "librbd/api/Mirror.h" +#include "tools/rbd_mirror/ClusterWatcher.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Types.h" +#include "test/rbd_mirror/test_fixture.h" +#include "test/librados/test_cxx.h" +#include "test/librbd/test_support.h" +#include "gtest/gtest.h" +#include <boost/scope_exit.hpp> +#include <iostream> +#include <map> +#include <memory> +#include <set> + +using rbd::mirror::ClusterWatcher; +using rbd::mirror::PeerSpec; +using rbd::mirror::RadosRef; +using std::map; +using std::set; +using std::string; + +void register_test_cluster_watcher() { +} + +class TestClusterWatcher : public ::rbd::mirror::TestFixture { +public: + + TestClusterWatcher() { + m_cluster = std::make_shared<librados::Rados>(); + EXPECT_EQ("", connect_cluster_pp(*m_cluster)); + } + + ~TestClusterWatcher() override { + m_cluster->wait_for_latest_osdmap(); + for (auto& pool : m_pools) { + EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str())); + } + } + + void SetUp() override { + TestFixture::SetUp(); + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + m_cluster, + m_threads)); + m_cluster_watcher.reset(new ClusterWatcher(m_cluster, m_lock, + m_service_daemon.get())); + } + + void TearDown() override { + m_service_daemon.reset(); + m_cluster_watcher.reset(); + TestFixture::TearDown(); + } + + void create_pool(bool enable_mirroring, const PeerSpec &peer, + string *uuid = nullptr, string *name=nullptr) { + string pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str())); + + int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); + ASSERT_GE(pool_id, 0); + + librados::IoCtx ioctx; + ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); + ioctx.application_enable("rbd", true); + + m_pools.insert(pool_name); + if (enable_mirroring) { + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, + RBD_MIRROR_MODE_POOL)); + + std::string gen_uuid; + ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add( + ioctx, uuid != nullptr ? uuid : &gen_uuid, + RBD_MIRROR_PEER_DIRECTION_RX_TX, + peer.cluster_name, peer.client_name)); + m_pool_peers[pool_id].insert(peer); + } + if (name != nullptr) { + *name = pool_name; + } + } + + void delete_pool(const string &name, const PeerSpec &peer) { + int64_t pool_id = m_cluster->pool_lookup(name.c_str()); + ASSERT_GE(pool_id, 0); + if (m_pool_peers.find(pool_id) != m_pool_peers.end()) { + m_pool_peers[pool_id].erase(peer); + if (m_pool_peers[pool_id].empty()) { + m_pool_peers.erase(pool_id); + } + } + m_pools.erase(name); + ASSERT_EQ(0, m_cluster->pool_delete(name.c_str())); + } + + void set_peer_config_key(const std::string& pool_name, + const PeerSpec &peer) { + int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); + ASSERT_GE(pool_id, 0); + + std::string json = + "{" + "\\\"mon_host\\\": \\\"" + peer.mon_host + "\\\", " + "\\\"key\\\": \\\"" + peer.key + "\\\"" + "}"; + + bufferlist in_bl; + ASSERT_EQ(0, m_cluster->mon_command( + "{" + "\"prefix\": \"config-key set\"," + "\"key\": \"" RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) + + "/" + peer.uuid + "\"," + "\"val\": \"" + json + "\"" + + "}", in_bl, nullptr, nullptr)); + } + + void create_cache_pool(const string &base_pool, string *cache_pool_name) { + bufferlist inbl; + *cache_pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, m_cluster->pool_create(cache_pool_name->c_str())); + + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool + + "\", \"tierpool\": \"" + *cache_pool_name + + "\", \"force_nonempty\": \"--force-nonempty\" }", + inbl, NULL, NULL)); + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool + + "\", \"overlaypool\": \"" + *cache_pool_name + "\"}", + inbl, NULL, NULL)); + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + *cache_pool_name + + "\", \"mode\": \"writeback\"}", + inbl, NULL, NULL)); + m_cluster->wait_for_latest_osdmap(); + } + + void remove_cache_pool(const string &base_pool, const string &cache_pool) { + bufferlist inbl; + // tear down tiers + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool + + "\"}", + inbl, NULL, NULL)); + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool + + "\", \"tierpool\": \"" + cache_pool + "\"}", + inbl, NULL, NULL)); + m_cluster->wait_for_latest_osdmap(); + m_cluster->pool_delete(cache_pool.c_str()); + } + + void check_peers() { + m_cluster_watcher->refresh_pools(); + std::lock_guard l{m_lock}; + ASSERT_EQ(m_pool_peers, m_cluster_watcher->get_pool_peers()); + } + + RadosRef m_cluster; + ceph::mutex m_lock = ceph::make_mutex("TestClusterWatcherLock"); + unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon; + unique_ptr<ClusterWatcher> m_cluster_watcher; + + set<string> m_pools; + ClusterWatcher::PoolPeers m_pool_peers; +}; + +TEST_F(TestClusterWatcher, NoPools) { + check_peers(); +} + +TEST_F(TestClusterWatcher, NoMirroredPools) { + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); +} + +TEST_F(TestClusterWatcher, ReplicatedPools) { + PeerSpec site1("", "site1", "mirror1"); + PeerSpec site2("", "site2", "mirror2"); + string first_pool, last_pool; + check_peers(); + create_pool(true, site1, &site1.uuid, &first_pool); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(true, site2, &site2.uuid); + check_peers(); + create_pool(true, site2, &site2.uuid); + check_peers(); + create_pool(true, site2, &site2.uuid, &last_pool); + check_peers(); + delete_pool(first_pool, site1); + check_peers(); + delete_pool(last_pool, site2); + check_peers(); +} + +TEST_F(TestClusterWatcher, CachePools) { + PeerSpec site1("", "site1", "mirror1"); + string base1, base2, cache1, cache2; + create_pool(true, site1, &site1.uuid, &base1); + check_peers(); + + create_cache_pool(base1, &cache1); + BOOST_SCOPE_EXIT( base1, cache1, this_ ) { + this_->remove_cache_pool(base1, cache1); + } BOOST_SCOPE_EXIT_END; + check_peers(); + + create_pool(false, PeerSpec(), nullptr, &base2); + create_cache_pool(base2, &cache2); + BOOST_SCOPE_EXIT( base2, cache2, this_ ) { + this_->remove_cache_pool(base2, cache2); + } BOOST_SCOPE_EXIT_END; + check_peers(); +} + +TEST_F(TestClusterWatcher, ConfigKey) { + REQUIRE(!is_librados_test_stub(*m_cluster)); + + std::string pool_name; + check_peers(); + + PeerSpec site1("", "site1", "mirror1"); + create_pool(true, site1, &site1.uuid, &pool_name); + check_peers(); + + PeerSpec site2("", "site2", "mirror2"); + site2.mon_host = "abc"; + site2.key = "xyz"; + create_pool(false, site2, &site2.uuid); + set_peer_config_key(pool_name, site2); + + check_peers(); +} + +TEST_F(TestClusterWatcher, SiteName) { + REQUIRE(!is_librados_test_stub(*m_cluster)); + + std::string site_name; + librbd::RBD rbd; + ASSERT_EQ(0, rbd.mirror_site_name_get(*m_cluster, &site_name)); + + m_cluster_watcher->refresh_pools(); + + std::lock_guard l{m_lock}; + ASSERT_EQ(site_name, m_cluster_watcher->get_site_name()); +} diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc new file mode 100644 index 000000000..5fa5d6db5 --- /dev/null +++ b/src/test/rbd_mirror/test_ImageDeleter.cc @@ -0,0 +1,313 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2016 SUSE LINUX GmbH + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "include/stringify.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Throttler.h" +#include "tools/rbd_mirror/Types.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Journal.h" +#include "librbd/internal.h" +#include "librbd/Utils.h" +#include "librbd/api/Image.h" +#include "librbd/api/Mirror.h" +#include "librbd/journal/DisabledPolicy.h" +#include "test/rbd_mirror/test_fixture.h" + +#include "test/librados/test.h" +#include "gtest/gtest.h" + +#define GLOBAL_IMAGE_ID "global_image_id" +#define GLOBAL_CLONE_IMAGE_ID "global_image_id_clone" + +#define dout_subsys ceph_subsys_rbd_mirror + +using rbd::mirror::RadosRef; +using rbd::mirror::TestFixture; +using namespace librbd; +using cls::rbd::MirrorImageMode; +using cls::rbd::MirrorImageState; + + +void register_test_rbd_mirror_image_deleter() { +} + +class TestImageDeleter : public TestFixture { +public: + const std::string m_local_mirror_uuid = "local mirror uuid"; + const std::string m_remote_mirror_uuid = "remote mirror uuid"; + + void SetUp() override { + TestFixture::SetUp(); + + m_image_deletion_throttler.reset( + new rbd::mirror::Throttler<>(g_ceph_context, + "rbd_mirror_concurrent_image_deletions")); + + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + _rados, m_threads)); + + librbd::api::Mirror<>::mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE); + + m_deleter = new rbd::mirror::ImageDeleter<>( + m_local_io_ctx, m_threads, m_image_deletion_throttler.get(), + m_service_daemon.get()); + + m_local_image_id = librbd::util::generate_image_id(m_local_io_ctx); + librbd::ImageOptions image_opts; + image_opts.set(RBD_IMAGE_OPTION_FEATURES, RBD_FEATURES_ALL); + EXPECT_EQ(0, librbd::create(m_local_io_ctx, m_image_name, m_local_image_id, + 1 << 20, image_opts, GLOBAL_IMAGE_ID, + m_remote_mirror_uuid, true)); + + cls::rbd::MirrorImage mirror_image( + MirrorImageMode::MIRROR_IMAGE_MODE_JOURNAL, GLOBAL_IMAGE_ID, + MirrorImageState::MIRROR_IMAGE_STATE_ENABLED); + EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, m_local_image_id, + mirror_image)); + } + + void TearDown() override { + remove_image(); + + C_SaferCond ctx; + m_deleter->shut_down(&ctx); + ctx.wait(); + + delete m_deleter; + m_service_daemon.reset(); + + TestFixture::TearDown(); + } + + void init_image_deleter() { + C_SaferCond ctx; + m_deleter->init(&ctx); + ASSERT_EQ(0, ctx.wait()); + } + + void remove_image() { + cls::rbd::MirrorImage mirror_image; + int r = cls_client::mirror_image_get(&m_local_io_ctx, m_local_image_id, + &mirror_image); + EXPECT_EQ(1, r == 0 || r == -ENOENT); + if (r != -ENOENT) { + mirror_image.state = MirrorImageState::MIRROR_IMAGE_STATE_ENABLED; + EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, + m_local_image_id, + mirror_image)); + } + promote_image(); + + NoOpProgressContext ctx; + r = librbd::api::Image<>::remove(m_local_io_ctx, m_image_name, ctx); + EXPECT_EQ(1, r == 0 || r == -ENOENT); + } + + void promote_image(ImageCtx *ictx=nullptr) { + bool close = false; + int r = 0; + if (!ictx) { + ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + r = ictx->state->open(0); + close = (r == 0); + } + + EXPECT_EQ(1, r == 0 || r == -ENOENT); + + if (r == 0) { + int r2 = librbd::api::Mirror<>::image_promote(ictx, true); + EXPECT_EQ(1, r2 == 0 || r2 == -EINVAL); + } + + if (close) { + EXPECT_EQ(0, ictx->state->close()); + } + } + + void demote_image(ImageCtx *ictx=nullptr) { + bool close = false; + if (!ictx) { + ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(0, ictx->state->open(0)); + close = true; + } + + EXPECT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); + + if (close) { + EXPECT_EQ(0, ictx->state->close()); + } + } + + void create_snapshot(std::string snap_name="snap1", bool protect=false) { + ImageCtx *ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(0, ictx->state->open(0)); + { + std::unique_lock image_locker{ictx->image_lock}; + ictx->set_journal_policy(new librbd::journal::DisabledPolicy()); + } + + librbd::NoOpProgressContext prog_ctx; + EXPECT_EQ(0, ictx->operations->snap_create( + cls::rbd::UserSnapshotNamespace(), snap_name, 0, prog_ctx)); + + if (protect) { + EXPECT_EQ(0, ictx->operations->snap_protect( + cls::rbd::UserSnapshotNamespace(), snap_name)); + } + + EXPECT_EQ(0, ictx->state->close()); + } + + std::string create_clone() { + ImageCtx *ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(0, ictx->state->open(0)); + { + std::unique_lock image_locker{ictx->image_lock}; + ictx->set_journal_policy(new librbd::journal::DisabledPolicy()); + } + + librbd::NoOpProgressContext prog_ctx; + EXPECT_EQ(0, ictx->operations->snap_create( + cls::rbd::UserSnapshotNamespace(), "snap1", 0, prog_ctx)); + EXPECT_EQ(0, ictx->operations->snap_protect( + cls::rbd::UserSnapshotNamespace(), "snap1")); + EXPECT_EQ(0, librbd::api::Image<>::snap_set( + ictx, cls::rbd::UserSnapshotNamespace(), "snap1")); + + std::string clone_id = librbd::util::generate_image_id(m_local_io_ctx); + librbd::ImageOptions clone_opts; + clone_opts.set(RBD_IMAGE_OPTION_FEATURES, ictx->features); + EXPECT_EQ(0, librbd::clone(m_local_io_ctx, m_local_image_id.c_str(), + nullptr, "snap1", m_local_io_ctx, + clone_id.c_str(), "clone1", clone_opts, + GLOBAL_CLONE_IMAGE_ID, m_remote_mirror_uuid)); + + cls::rbd::MirrorImage mirror_image( + MirrorImageMode::MIRROR_IMAGE_MODE_JOURNAL, GLOBAL_CLONE_IMAGE_ID, + MirrorImageState::MIRROR_IMAGE_STATE_ENABLED); + EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, clone_id, + mirror_image)); + EXPECT_EQ(0, ictx->state->close()); + return clone_id; + } + + void check_image_deleted() { + ImageCtx *ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(-ENOENT, ictx->state->open(0)); + + cls::rbd::MirrorImage mirror_image; + EXPECT_EQ(-ENOENT, cls_client::mirror_image_get(&m_local_io_ctx, + m_local_image_id, + &mirror_image)); + } + + int trash_move(const std::string& global_image_id) { + C_SaferCond ctx; + rbd::mirror::ImageDeleter<>::trash_move(m_local_io_ctx, global_image_id, + true, m_threads->work_queue, &ctx); + return ctx.wait(); + } + + librbd::RBD rbd; + std::string m_local_image_id; + std::unique_ptr<rbd::mirror::Throttler<>> m_image_deletion_throttler; + std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon; + rbd::mirror::ImageDeleter<> *m_deleter; +}; + +TEST_F(TestImageDeleter, ExistingTrashMove) { + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + init_image_deleter(); + + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestImageDeleter, LiveTrashMove) { + init_image_deleter(); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestImageDeleter, Delete_Image_With_Snapshots) { + init_image_deleter(); + create_snapshot("snap1"); + create_snapshot("snap2"); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + EXPECT_EQ(0, ctx.wait()); + + ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); + ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); +} + +TEST_F(TestImageDeleter, Delete_Image_With_ProtectedSnapshots) { + init_image_deleter(); + create_snapshot("snap1", true); + create_snapshot("snap2", true); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + EXPECT_EQ(0, ctx.wait()); + + ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); + ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); +} + +TEST_F(TestImageDeleter, Delete_Image_With_Clone) { + init_image_deleter(); + std::string clone_id = create_clone(); + + C_SaferCond ctx1; + m_deleter->set_busy_timer_interval(0.1); + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx1); + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + EXPECT_EQ(-EBUSY, ctx1.wait()); + + C_SaferCond ctx2; + m_deleter->wait_for_deletion(clone_id, false, &ctx2); + ASSERT_EQ(0, trash_move(GLOBAL_CLONE_IMAGE_ID)); + EXPECT_EQ(0, ctx2.wait()); + + C_SaferCond ctx3; + m_deleter->wait_for_deletion(m_local_image_id, true, &ctx3); + EXPECT_EQ(0, ctx3.wait()); + + ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); + ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); +} + diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc new file mode 100644 index 000000000..76a6919d5 --- /dev/null +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -0,0 +1,1664 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph distributed storage system + * + * Copyright (C) 2016 Mirantis Inc + * + * Author: Mykola Golub <mgolub@mirantis.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "include/stringify.h" +#include "test/librbd/test_support.h" +#include "test/rbd_mirror/test_fixture.h" +#include "cls/journal/cls_journal_types.h" +#include "cls/journal/cls_journal_client.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "journal/Journaler.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/internal.h" +#include "librbd/api/Io.h" +#include "librbd/api/Mirror.h" +#include "librbd/api/Snapshot.h" +#include "librbd/io/AioCompletion.h" +#include "librbd/io/ReadResult.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/MirrorStatusUpdater.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Throttler.h" +#include "tools/rbd_mirror/Types.h" + +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" + +void register_test_rbd_mirror() { +} + +#define TEST_IO_SIZE 512 +#define TEST_IO_COUNT 11 + +namespace rbd { +namespace mirror { + +template <typename T> +class TestImageReplayer : public TestFixture { +public: + static const cls::rbd::MirrorImageMode MIRROR_IMAGE_MODE = + T::MIRROR_IMAGE_MODE; + static const uint64_t FEATURES = T::FEATURES; + + struct C_WatchCtx : public librados::WatchCtx2 { + TestImageReplayer *test; + std::string oid; + ceph::mutex lock = ceph::make_mutex("C_WatchCtx::lock"); + ceph::condition_variable cond; + bool notified; + + C_WatchCtx(TestImageReplayer *test, const std::string &oid) + : test(test), oid(oid), notified(false) { + } + + void handle_notify(uint64_t notify_id, uint64_t cookie, + uint64_t notifier_id, bufferlist& bl_) override { + bufferlist bl; + test->m_remote_ioctx.notify_ack(oid, notify_id, cookie, bl); + + std::lock_guard locker{lock}; + notified = true; + cond.notify_all(); + } + + void handle_error(uint64_t cookie, int err) override { + ASSERT_EQ(0, err); + } + }; + + TestImageReplayer() + : m_local_cluster(new librados::Rados()), m_watch_handle(0) + { + EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get())); + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false")); + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1")); + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_commit_age", + "0.1")); + m_local_pool_name = get_temp_pool_name(); + EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str())); + EXPECT_EQ(0, m_local_cluster->ioctx_create(m_local_pool_name.c_str(), + m_local_ioctx)); + m_local_ioctx.application_enable("rbd", true); + + EXPECT_EQ("", connect_cluster_pp(m_remote_cluster)); + EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false")); + + m_remote_pool_name = get_temp_pool_name(); + EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str())); + m_remote_pool_id = m_remote_cluster.pool_lookup(m_remote_pool_name.c_str()); + EXPECT_GE(m_remote_pool_id, 0); + + EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(), + m_remote_ioctx)); + m_remote_ioctx.application_enable("rbd", true); + + // make snap id debugging easier when local/remote have different mappings + uint64_t snap_id; + EXPECT_EQ(0, m_remote_ioctx.selfmanaged_snap_create(&snap_id)); + + uint64_t features = FEATURES; + if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_POOL)); + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_local_ioctx, + RBD_MIRROR_MODE_POOL)); + } else { + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_IMAGE)); + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_local_ioctx, + RBD_MIRROR_MODE_IMAGE)); + + + uuid_d uuid_gen; + uuid_gen.generate_random(); + std::string remote_peer_uuid = uuid_gen.to_string(); + + EXPECT_EQ(0, librbd::cls_client::mirror_peer_add( + &m_remote_ioctx, {remote_peer_uuid, + cls::rbd::MIRROR_PEER_DIRECTION_RX_TX, + "siteA", "client", m_local_mirror_uuid})); + + m_pool_meta_cache.set_remote_pool_meta( + m_remote_ioctx.get_id(), {m_remote_mirror_uuid, remote_peer_uuid}); + } + + EXPECT_EQ(0, librbd::api::Mirror<>::uuid_get(m_remote_ioctx, + &m_remote_mirror_uuid)); + EXPECT_EQ(0, librbd::api::Mirror<>::uuid_get(m_local_ioctx, + &m_local_mirror_uuid)); + + m_image_name = get_temp_image_name(); + int order = 0; + EXPECT_EQ(0, librbd::create(m_remote_ioctx, m_image_name.c_str(), 1 << 22, + false, features, &order, 0, 0)); + + if (MIRROR_IMAGE_MODE != cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + librbd::ImageCtx* remote_image_ctx; + open_remote_image(&remote_image_ctx); + EXPECT_EQ(0, + librbd::api::Mirror<>::image_enable( + remote_image_ctx, + static_cast<rbd_mirror_image_mode_t>(MIRROR_IMAGE_MODE), + false)); + close_image(remote_image_ctx); + } + + m_remote_image_id = get_image_id(m_remote_ioctx, m_image_name); + m_global_image_id = get_global_image_id(m_remote_ioctx, m_remote_image_id); + + auto cct = reinterpret_cast<CephContext*>(m_local_ioctx.cct()); + m_threads.reset(new Threads<>(m_local_cluster)); + + m_image_sync_throttler.reset(new Throttler<>( + cct, "rbd_mirror_concurrent_image_syncs")); + + m_instance_watcher = InstanceWatcher<>::create( + m_local_ioctx, *m_threads->asio_engine, nullptr, + m_image_sync_throttler.get()); + m_instance_watcher->handle_acquire_leader(); + + EXPECT_EQ(0, m_local_ioctx.create(RBD_MIRRORING, false)); + + m_local_status_updater = MirrorStatusUpdater<>::create( + m_local_ioctx, m_threads.get(), ""); + C_SaferCond status_updater_ctx; + m_local_status_updater->init(&status_updater_ctx); + EXPECT_EQ(0, status_updater_ctx.wait()); + } + + ~TestImageReplayer() override + { + unwatch(); + + m_instance_watcher->handle_release_leader(); + + delete m_replayer; + delete m_instance_watcher; + + C_SaferCond status_updater_ctx; + m_local_status_updater->shut_down(&status_updater_ctx); + EXPECT_EQ(0, status_updater_ctx.wait()); + delete m_local_status_updater; + + EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str())); + EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str())); + } + + void create_replayer() { + m_replayer = new ImageReplayer<>(m_local_ioctx, m_local_mirror_uuid, + m_global_image_id, m_threads.get(), + m_instance_watcher, m_local_status_updater, + nullptr, &m_pool_meta_cache); + m_replayer->add_peer({"peer uuid", m_remote_ioctx, + {m_remote_mirror_uuid, "remote mirror peer uuid"}, + nullptr}); + } + + void start() + { + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + + create_watch_ctx(); + } + + void create_watch_ctx() { + std::string oid; + if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + oid = ::journal::Journaler::header_oid(m_remote_image_id); + } else { + oid = librbd::util::header_name(m_remote_image_id); + } + + ASSERT_EQ(0U, m_watch_handle); + ASSERT_TRUE(m_watch_ctx == nullptr); + m_watch_ctx = new C_WatchCtx(this, oid); + ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx)); + } + + void unwatch() { + if (m_watch_handle != 0) { + m_remote_ioctx.unwatch2(m_watch_handle); + delete m_watch_ctx; + m_watch_ctx = nullptr; + m_watch_handle = 0; + } + } + + void stop() + { + unwatch(); + + C_SaferCond cond; + m_replayer->stop(&cond); + ASSERT_EQ(0, cond.wait()); + } + + void bootstrap() + { + create_replayer(); + + start(); + wait_for_replay_complete(); + stop(); + } + + std::string get_temp_image_name() + { + return "image" + stringify(++_image_number); + } + + std::string get_image_id(librados::IoCtx &ioctx, const string &image_name) + { + std::string obj = librbd::util::id_obj_name(image_name); + std::string id; + EXPECT_EQ(0, librbd::cls_client::get_id(&ioctx, obj, &id)); + return id; + } + + std::string get_global_image_id(librados::IoCtx& io_ctx, + const std::string& image_id) { + cls::rbd::MirrorImage mirror_image; + EXPECT_EQ(0, librbd::cls_client::mirror_image_get(&io_ctx, image_id, + &mirror_image)); + return mirror_image.global_image_id; + } + + void open_image(librados::IoCtx &ioctx, const std::string &image_name, + bool readonly, librbd::ImageCtx **ictxp) + { + librbd::ImageCtx *ictx = new librbd::ImageCtx(image_name.c_str(), + "", "", ioctx, readonly); + EXPECT_EQ(0, ictx->state->open(0)); + *ictxp = ictx; + } + + void open_local_image(librbd::ImageCtx **ictxp) + { + open_image(m_local_ioctx, m_image_name, true, ictxp); + } + + void open_remote_image(librbd::ImageCtx **ictxp) + { + open_image(m_remote_ioctx, m_image_name, false, ictxp); + } + + void close_image(librbd::ImageCtx *ictx) + { + ictx->state->close(); + } + + void get_commit_positions(cls::journal::ObjectPosition *master_position, + cls::journal::ObjectPosition *mirror_position) + { + std::string master_client_id = ""; + std::string mirror_client_id = m_local_mirror_uuid; + + m_replayer->flush(); + + C_SaferCond cond; + uint64_t minimum_set; + uint64_t active_set; + std::set<cls::journal::Client> registered_clients; + std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); + cls::journal::client::get_mutable_metadata(m_remote_ioctx, oid, + &minimum_set, &active_set, + ®istered_clients, &cond); + ASSERT_EQ(0, cond.wait()); + + *master_position = cls::journal::ObjectPosition(); + *mirror_position = cls::journal::ObjectPosition(); + + std::set<cls::journal::Client>::const_iterator c; + for (c = registered_clients.begin(); c != registered_clients.end(); ++c) { + std::cout << __func__ << ": client: " << *c << std::endl; + if (c->state != cls::journal::CLIENT_STATE_CONNECTED) { + continue; + } + cls::journal::ObjectPositions object_positions = + c->commit_position.object_positions; + cls::journal::ObjectPositions::const_iterator p = + object_positions.begin(); + if (p != object_positions.end()) { + if (c->id == master_client_id) { + ASSERT_EQ(cls::journal::ObjectPosition(), *master_position); + *master_position = *p; + } else if (c->id == mirror_client_id) { + ASSERT_EQ(cls::journal::ObjectPosition(), *mirror_position); + *mirror_position = *p; + } + } + } + } + + bool wait_for_watcher_notify(int seconds) + { + if (m_watch_handle == 0) { + return false; + } + + std::unique_lock locker{m_watch_ctx->lock}; + while (!m_watch_ctx->notified) { + if (m_watch_ctx->cond.wait_for(locker, + std::chrono::seconds(seconds)) == + std::cv_status::timeout) { + return false; + } + } + m_watch_ctx->notified = false; + return true; + } + + int get_last_mirror_snapshot(librados::IoCtx& io_ctx, + const std::string& image_id, + uint64_t* mirror_snap_id, + cls::rbd::MirrorSnapshotNamespace* mirror_ns) { + auto header_oid = librbd::util::header_name(image_id); + ::SnapContext snapc; + int r = librbd::cls_client::get_snapcontext(&io_ctx, header_oid, &snapc); + if (r < 0) { + return r; + } + + // stored in reverse order + for (auto snap_id : snapc.snaps) { + cls::rbd::SnapshotInfo snap_info; + r = librbd::cls_client::snapshot_get(&io_ctx, header_oid, snap_id, + &snap_info); + if (r < 0) { + return r; + } + + auto ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &snap_info.snapshot_namespace); + if (ns != nullptr) { + *mirror_snap_id = snap_id; + *mirror_ns = *ns; + return 0; + } + } + + return -ENOENT; + } + + void wait_for_journal_synced() { + cls::journal::ObjectPosition master_position; + cls::journal::ObjectPosition mirror_position; + for (int i = 0; i < 100; i++) { + get_commit_positions(&master_position, &mirror_position); + if (master_position == mirror_position) { + break; + } + wait_for_watcher_notify(1); + } + + ASSERT_EQ(master_position, mirror_position); + } + + void wait_for_snapshot_synced() { + uint64_t remote_snap_id = CEPH_NOSNAP; + cls::rbd::MirrorSnapshotNamespace remote_mirror_ns; + ASSERT_EQ(0, get_last_mirror_snapshot(m_remote_ioctx, m_remote_image_id, + &remote_snap_id, &remote_mirror_ns)); + + std::cout << "remote_snap_id=" << remote_snap_id << std::endl; + + std::string local_image_id; + ASSERT_EQ(0, librbd::cls_client::mirror_image_get_image_id( + &m_local_ioctx, m_global_image_id, &local_image_id)); + + uint64_t local_snap_id = CEPH_NOSNAP; + cls::rbd::MirrorSnapshotNamespace local_mirror_ns; + for (int i = 0; i < 100; i++) { + int r = get_last_mirror_snapshot(m_local_ioctx, local_image_id, + &local_snap_id, &local_mirror_ns); + if (r == 0 && + ((remote_mirror_ns.state == + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY && + local_mirror_ns.state == + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) || + (remote_mirror_ns.state == + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED && + local_mirror_ns.state == + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED)) && + local_mirror_ns.primary_mirror_uuid == m_remote_mirror_uuid && + local_mirror_ns.primary_snap_id == remote_snap_id && + local_mirror_ns.complete) { + + std::cout << "local_snap_id=" << local_snap_id << ", " + << "local_snap_ns=" << local_mirror_ns << std::endl; + return; + } + + wait_for_watcher_notify(1); + } + + ADD_FAILURE() << "failed to locate matching snapshot: " + << "remote_snap_id=" << remote_snap_id << ", " + << "remote_snap_ns=" << remote_mirror_ns << ", " + << "local_snap_id=" << local_snap_id << ", " + << "local_snap_ns=" << local_mirror_ns; + } + + void wait_for_replay_complete() + { + if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + wait_for_journal_synced(); + } else { + wait_for_snapshot_synced(); + } + } + + void wait_for_stopped() { + for (int i = 0; i < 100; i++) { + if (m_replayer->is_stopped()) { + break; + } + wait_for_watcher_notify(1); + } + ASSERT_TRUE(m_replayer->is_stopped()); + } + + void write_test_data(librbd::ImageCtx *ictx, const char *test_data, off_t off, + size_t len) + { + size_t written; + bufferlist bl; + bl.append(std::string(test_data, len)); + written = librbd::api::Io<>::write(*ictx, off, len, std::move(bl), 0); + printf("wrote: %d\n", (int)written); + ASSERT_EQ(len, written); + } + + void read_test_data(librbd::ImageCtx *ictx, const char *expected, off_t off, + size_t len) + { + ssize_t read; + char *result = (char *)malloc(len + 1); + + ASSERT_NE(static_cast<char *>(NULL), result); + read = librbd::api::Io<>::read( + *ictx, off, len, librbd::io::ReadResult{result, len}, 0); + printf("read: %d\n", (int)read); + ASSERT_EQ(len, static_cast<size_t>(read)); + result[len] = '\0'; + if (memcmp(result, expected, len)) { + printf("read: %s\nexpected: %s\n", result, expected); + ASSERT_EQ(0, memcmp(result, expected, len)); + } + free(result); + } + + void generate_test_data() { + for (int i = 0; i < TEST_IO_SIZE; ++i) { + m_test_data[i] = (char) (rand() % (126 - 33) + 33); + } + m_test_data[TEST_IO_SIZE] = '\0'; + } + + void flush(librbd::ImageCtx *ictx) + { + C_SaferCond aio_flush_ctx; + auto c = librbd::io::AioCompletion::create(&aio_flush_ctx); + c->get(); + librbd::api::Io<>::aio_flush(*ictx, c, true); + ASSERT_EQ(0, c->wait_for_complete()); + c->put(); + + if (MIRROR_IMAGE_MODE == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + C_SaferCond journal_flush_ctx; + ictx->journal->flush_commit_position(&journal_flush_ctx); + ASSERT_EQ(0, journal_flush_ctx.wait()); + } else { + uint64_t snap_id = CEPH_NOSNAP; + ASSERT_EQ(0, librbd::api::Mirror<>::image_snapshot_create( + ictx, 0, &snap_id)); + } + + printf("flushed\n"); + } + + static int _image_number; + + PoolMetaCache m_pool_meta_cache{g_ceph_context}; + + std::shared_ptr<librados::Rados> m_local_cluster; + std::unique_ptr<Threads<>> m_threads; + std::unique_ptr<Throttler<>> m_image_sync_throttler; + librados::Rados m_remote_cluster; + InstanceWatcher<> *m_instance_watcher; + MirrorStatusUpdater<> *m_local_status_updater; + std::string m_local_mirror_uuid = "local mirror uuid"; + std::string m_remote_mirror_uuid = "remote mirror uuid"; + std::string m_local_pool_name, m_remote_pool_name; + librados::IoCtx m_local_ioctx, m_remote_ioctx; + std::string m_image_name; + int64_t m_remote_pool_id; + std::string m_remote_image_id; + std::string m_global_image_id; + ImageReplayer<> *m_replayer = nullptr; + C_WatchCtx *m_watch_ctx = nullptr; + uint64_t m_watch_handle = 0; + char m_test_data[TEST_IO_SIZE + 1]; + std::string m_journal_commit_age; +}; + +template <typename T> +int TestImageReplayer<T>::_image_number; + +template <cls::rbd::MirrorImageMode _mirror_image_mode, uint64_t _features> +class TestImageReplayerParams { +public: + static const cls::rbd::MirrorImageMode MIRROR_IMAGE_MODE = _mirror_image_mode; + static const uint64_t FEATURES = _features; +}; + +typedef ::testing::Types<TestImageReplayerParams< + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, 125>, + TestImageReplayerParams< + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 1>, + TestImageReplayerParams< + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 5>, + TestImageReplayerParams< + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 61>, + TestImageReplayerParams< + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, 125>> + TestImageReplayerTypes; + +TYPED_TEST_SUITE(TestImageReplayer, TestImageReplayerTypes); + +TYPED_TEST(TestImageReplayer, Bootstrap) +{ + this->bootstrap(); +} + +typedef TestImageReplayer<TestImageReplayerParams< + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, 125>> TestImageReplayerJournal; + +TYPED_TEST(TestImageReplayer, BootstrapErrorLocalImageExists) +{ + int order = 0; + EXPECT_EQ(0, librbd::create(this->m_local_ioctx, this->m_image_name.c_str(), + 1 << 22, false, 0, &order, 0, 0)); + + this->create_replayer(); + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(-EEXIST, cond.wait()); +} + +TEST_F(TestImageReplayerJournal, BootstrapErrorNoJournal) +{ + ASSERT_EQ(0, librbd::Journal<>::remove(this->m_remote_ioctx, + this->m_remote_image_id)); + + this->create_replayer(); + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(-ENOENT, cond.wait()); +} + +TYPED_TEST(TestImageReplayer, BootstrapErrorMirrorDisabled) +{ + // disable remote image mirroring + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(this->m_remote_ioctx, + RBD_MIRROR_MODE_IMAGE)); + librbd::ImageCtx *ictx; + this->open_remote_image(&ictx); + ASSERT_EQ(0, librbd::api::Mirror<>::image_disable(ictx, true)); + this->close_image(ictx); + + this->create_replayer(); + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(-ENOENT, cond.wait()); +} + +TYPED_TEST(TestImageReplayer, BootstrapMirrorDisabling) +{ + // set remote image mirroring state to DISABLING + if (gtest_TypeParam_::MIRROR_IMAGE_MODE == + cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(this->m_remote_ioctx, + RBD_MIRROR_MODE_IMAGE)); + librbd::ImageCtx *ictx; + this->open_remote_image(&ictx); + ASSERT_EQ(0, librbd::api::Mirror<>::image_enable( + ictx, RBD_MIRROR_IMAGE_MODE_JOURNAL, false)); + this->close_image(ictx); + } + + cls::rbd::MirrorImage mirror_image; + ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&this->m_remote_ioctx, + this->m_remote_image_id, + &mirror_image)); + mirror_image.state = cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING; + ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&this->m_remote_ioctx, + this->m_remote_image_id, + mirror_image)); + + this->create_replayer(); + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(-ENOENT, cond.wait()); + ASSERT_TRUE(this->m_replayer->is_stopped()); +} + +TYPED_TEST(TestImageReplayer, BootstrapDemoted) +{ + // demote remote image + librbd::ImageCtx *ictx; + this->open_remote_image(&ictx); + ASSERT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); + this->close_image(ictx); + + this->create_replayer(); + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(-EREMOTEIO, cond.wait()); + ASSERT_TRUE(this->m_replayer->is_stopped()); +} + +TYPED_TEST(TestImageReplayer, StartInterrupted) +{ + this->create_replayer(); + C_SaferCond start_cond, stop_cond; + this->m_replayer->start(&start_cond); + this->m_replayer->stop(&stop_cond); + int r = start_cond.wait(); + printf("start returned %d\n", r); + // TODO: improve the test to avoid this race + ASSERT_TRUE(r == -ECANCELED || r == 0); + ASSERT_EQ(0, stop_cond.wait()); +} + +TEST_F(TestImageReplayerJournal, JournalReset) +{ + this->bootstrap(); + delete this->m_replayer; + + ASSERT_EQ(0, librbd::Journal<>::reset(this->m_remote_ioctx, + this->m_remote_image_id)); + + // try to recover + this->bootstrap(); +} + +TEST_F(TestImageReplayerJournal, ErrorNoJournal) +{ + this->bootstrap(); + + // disable remote journal journaling + // (reset before disabling, so it does not fail with EBUSY) + ASSERT_EQ(0, librbd::Journal<>::reset(this->m_remote_ioctx, + this->m_remote_image_id)); + librbd::ImageCtx *ictx; + this->open_remote_image(&ictx); + uint64_t features; + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0, ictx->operations->update_features(RBD_FEATURE_JOURNALING, + false)); + this->close_image(ictx); + + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); +} + +TYPED_TEST(TestImageReplayer, StartStop) +{ + this->bootstrap(); + + this->start(); + this->wait_for_replay_complete(); + this->stop(); +} + +TYPED_TEST(TestImageReplayer, WriteAndStartReplay) +{ + this->bootstrap(); + + // Write to remote image and start replay + + librbd::ImageCtx *ictx; + + this->generate_test_data(); + this->open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + this->start(); + this->wait_for_replay_complete(); + this->stop(); + + this->open_local_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); +} + +TYPED_TEST(TestImageReplayer, StartReplayAndWrite) +{ + this->bootstrap(); + + // Start replay and write to remote image + + librbd::ImageCtx *ictx; + + this->start(); + + this->generate_test_data(); + this->open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + + this->wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + + this->stop(); +} + +TEST_F(TestImageReplayerJournal, NextTag) +{ + this->bootstrap(); + + // write, reopen, and write again to test switch to the next tag + + librbd::ImageCtx *ictx; + + this->start(); + + this->generate_test_data(); + + const int N = 10; + + for (int j = 0; j < N; j++) { + this->open_remote_image(&ictx); + for (int i = j * TEST_IO_COUNT; i < (j + 1) * TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + } + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + for (int i = 0; i < N * TEST_IO_COUNT; ++i) { + this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + + this->stop(); +} + +TYPED_TEST(TestImageReplayer, Resync) +{ + this->bootstrap(); + + librbd::ImageCtx *ictx; + + this->start(); + + this->generate_test_data(); + + this->open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + + this->wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + this->open_local_image(&ictx); + EXPECT_EQ(0, librbd::api::Mirror<>::image_resync(ictx)); + this->close_image(ictx); + + this->wait_for_stopped(); + + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + + ASSERT_TRUE(this->m_replayer->is_replaying()); + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + + this->stop(); +} + +TYPED_TEST(TestImageReplayer, Resync_While_Stop) +{ + this->bootstrap(); + + this->start(); + + this->generate_test_data(); + + librbd::ImageCtx *ictx; + this->open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + + this->wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + C_SaferCond cond; + this->m_replayer->stop(&cond); + ASSERT_EQ(0, cond.wait()); + + this->open_local_image(&ictx); + EXPECT_EQ(0, librbd::api::Mirror<>::image_resync(ictx)); + this->close_image(ictx); + + C_SaferCond cond2; + this->m_replayer->start(&cond2); + ASSERT_EQ(0, cond2.wait()); + + ASSERT_TRUE(this->m_replayer->is_stopped()); + + C_SaferCond cond3; + this->m_replayer->start(&cond3); + ASSERT_EQ(0, cond3.wait()); + + ASSERT_TRUE(this->m_replayer->is_replaying()); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + + this->stop(); +} + +TYPED_TEST(TestImageReplayer, Resync_StartInterrupted) +{ + this->bootstrap(); + + librbd::ImageCtx *ictx; + this->open_local_image(&ictx); + EXPECT_EQ(0, librbd::api::Mirror<>::image_resync(ictx)); + this->close_image(ictx); + + C_SaferCond cond; + this->m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + + ASSERT_TRUE(this->m_replayer->is_stopped()); + + C_SaferCond cond2; + this->m_replayer->start(&cond2); + ASSERT_EQ(0, cond2.wait()); + + this->create_watch_ctx(); + + ASSERT_TRUE(this->m_replayer->is_replaying()); + + this->generate_test_data(); + this->open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + + this->wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + + this->stop(); +} + +TEST_F(TestImageReplayerJournal, MultipleReplayFailures_SingleEpoch) { + this->bootstrap(); + + // inject a snapshot that cannot be unprotected + librbd::ImageCtx *ictx; + this->open_image(this->m_local_ioctx, this->m_image_name, false, &ictx); + ictx->features &= ~RBD_FEATURE_JOURNALING; + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + "foo", 0, prog_ctx)); + ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + "foo")); + ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, + {ictx->md_ctx.get_id(), "", + ictx->id, + ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), "foo"}]}, + "dummy child id")); + this->close_image(ictx); + + // race failed op shut down with new ops + this->open_remote_image(&ictx); + for (uint64_t i = 0; i < 10; ++i) { + std::shared_lock owner_locker{ictx->owner_lock}; + C_SaferCond request_lock; + ictx->exclusive_lock->acquire_lock(&request_lock); + ASSERT_EQ(0, request_lock.wait()); + + C_SaferCond append_ctx; + ictx->journal->append_op_event( + i, + librbd::journal::EventEntry{ + librbd::journal::SnapUnprotectEvent{i, + cls::rbd::UserSnapshotNamespace(), + "foo"}}, + &append_ctx); + ASSERT_EQ(0, append_ctx.wait()); + + C_SaferCond commit_ctx; + ictx->journal->commit_op_event(i, 0, &commit_ctx); + ASSERT_EQ(0, commit_ctx.wait()); + + C_SaferCond release_ctx; + ictx->exclusive_lock->release_lock(&release_ctx); + ASSERT_EQ(0, release_ctx.wait()); + } + + for (uint64_t i = 0; i < 5; ++i) { + this->start(); + this->wait_for_stopped(); + this->unwatch(); + } + this->close_image(ictx); +} + +TEST_F(TestImageReplayerJournal, MultipleReplayFailures_MultiEpoch) { + this->bootstrap(); + + // inject a snapshot that cannot be unprotected + librbd::ImageCtx *ictx; + this->open_image(this->m_local_ioctx, this->m_image_name, false, &ictx); + ictx->features &= ~RBD_FEATURE_JOURNALING; + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + "foo", 0, prog_ctx)); + ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + "foo")); + ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, + {ictx->md_ctx.get_id(), "", + ictx->id, + ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), + "foo"}]}, + "dummy child id")); + this->close_image(ictx); + + // race failed op shut down with new tag flush + this->open_remote_image(&ictx); + { + std::shared_lock owner_locker{ictx->owner_lock}; + C_SaferCond request_lock; + ictx->exclusive_lock->acquire_lock(&request_lock); + ASSERT_EQ(0, request_lock.wait()); + + C_SaferCond append_ctx; + ictx->journal->append_op_event( + 1U, + librbd::journal::EventEntry{ + librbd::journal::SnapUnprotectEvent{1U, + cls::rbd::UserSnapshotNamespace(), + "foo"}}, + &append_ctx); + ASSERT_EQ(0, append_ctx.wait()); + + C_SaferCond commit_ctx; + ictx->journal->commit_op_event(1U, 0, &commit_ctx); + ASSERT_EQ(0, commit_ctx.wait()); + + C_SaferCond release_ctx; + ictx->exclusive_lock->release_lock(&release_ctx); + ASSERT_EQ(0, release_ctx.wait()); + } + + this->generate_test_data(); + this->write_test_data(ictx, this->m_test_data, 0, TEST_IO_SIZE); + + for (uint64_t i = 0; i < 5; ++i) { + this->start(); + this->wait_for_stopped(); + this->unwatch(); + } + this->close_image(ictx); +} + +TEST_F(TestImageReplayerJournal, Disconnect) +{ + this->bootstrap(); + + // Make sure rbd_mirroring_resync_after_disconnect is not set + EXPECT_EQ(0, this->m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "false")); + + // Test start fails if disconnected + + librbd::ImageCtx *ictx; + + this->generate_test_data(); + this->open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + std::string oid = ::journal::Journaler::header_oid(this->m_remote_image_id); + ASSERT_EQ(0, + cls::journal::client::client_update_state(this->m_remote_ioctx, + oid, this->m_local_mirror_uuid, + cls::journal::CLIENT_STATE_DISCONNECTED)); + + C_SaferCond cond1; + this->m_replayer->start(&cond1); + ASSERT_EQ(-ENOTCONN, cond1.wait()); + + // Test start succeeds after resync + + this->open_local_image(&ictx); + librbd::Journal<>::request_resync(ictx); + this->close_image(ictx); + C_SaferCond cond2; + this->m_replayer->start(&cond2); + ASSERT_EQ(0, cond2.wait()); + + this->start(); + this->wait_for_replay_complete(); + + // Test replay stopped after disconnect + + this->open_remote_image(&ictx); + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + ASSERT_EQ(0, + cls::journal::client::client_update_state(this->m_remote_ioctx, oid, + this->m_local_mirror_uuid, + cls::journal::CLIENT_STATE_DISCONNECTED)); + bufferlist bl; + ASSERT_EQ(0, this->m_remote_ioctx.notify2(oid, bl, 5000, NULL)); + + this->wait_for_stopped(); + + // Test start fails after disconnect + + C_SaferCond cond3; + this->m_replayer->start(&cond3); + ASSERT_EQ(-ENOTCONN, cond3.wait()); + C_SaferCond cond4; + this->m_replayer->start(&cond4); + ASSERT_EQ(-ENOTCONN, cond4.wait()); + + // Test automatic resync if rbd_mirroring_resync_after_disconnect is set + + EXPECT_EQ(0, this->m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "true")); + + // Resync is flagged on first start attempt + C_SaferCond cond5; + this->m_replayer->start(&cond5); + ASSERT_EQ(-ENOTCONN, cond5.wait()); + + C_SaferCond cond6; + this->m_replayer->start(&cond6); + ASSERT_EQ(0, cond6.wait()); + this->wait_for_replay_complete(); + + this->stop(); +} + +TEST_F(TestImageReplayerJournal, UpdateFeatures) +{ + // TODO add support to snapshot-based mirroring + const uint64_t FEATURES_TO_UPDATE = + RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF; + + uint64_t features; + librbd::ImageCtx *ictx; + + // Make sure the features we will update are disabled initially + + this->open_remote_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + features &= FEATURES_TO_UPDATE; + if (features) { + ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, + false)); + } + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + this->close_image(ictx); + + this->bootstrap(); + + this->open_remote_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + this->close_image(ictx); + + this->open_local_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + this->close_image(ictx); + + // Start replay and update features + + this->start(); + + this->open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, + true)); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); + this->close_image(ictx); + + this->open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, + false)); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + this->close_image(ictx); + + // Test update_features error does not stop replication + + this->open_remote_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_NE(0U, features & RBD_FEATURE_EXCLUSIVE_LOCK); + ASSERT_EQ(-EINVAL, ictx->operations->update_features(RBD_FEATURE_EXCLUSIVE_LOCK, + false)); + this->generate_test_data(); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->read_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + + this->stop(); +} + +TEST_F(TestImageReplayerJournal, MetadataSetRemove) +{ + // TODO add support to snapshot-based mirroring + const std::string KEY = "test_key"; + const std::string VALUE = "test_value"; + + librbd::ImageCtx *ictx; + std::string value; + + this->bootstrap(); + + this->start(); + + // Test metadata_set replication + + this->open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->metadata_set(KEY, VALUE)); + value.clear(); + ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); + ASSERT_EQ(VALUE, value); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + value.clear(); + ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); + ASSERT_EQ(VALUE, value); + this->close_image(ictx); + + // Test metadata_remove replication + + this->open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->metadata_remove(KEY)); + ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); + this->close_image(ictx); + + this->wait_for_replay_complete(); + + this->open_local_image(&ictx); + ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); + this->close_image(ictx); + + this->stop(); +} + +TEST_F(TestImageReplayerJournal, MirroringDelay) +{ + // TODO add support to snapshot-based mirroring + const double DELAY = 10; // set less than wait_for_replay_complete timeout + + librbd::ImageCtx *ictx; + utime_t start_time; + double delay; + + this->bootstrap(); + + ASSERT_EQ(0, this->m_local_cluster->conf_set("rbd_mirroring_replay_delay", + stringify(DELAY).c_str())); + this->open_local_image(&ictx); + ASSERT_EQ(DELAY, ictx->mirroring_replay_delay); + this->close_image(ictx); + + this->start(); + + // Test delay + + this->generate_test_data(); + this->open_remote_image(&ictx); + start_time = ceph_clock_now(); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->flush(ictx); + this->close_image(ictx); + + this->wait_for_replay_complete(); + delay = ceph_clock_now() - start_time; + ASSERT_GE(delay, DELAY); + + // Test stop when delaying replay + + this->open_remote_image(&ictx); + start_time = ceph_clock_now(); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + this->write_test_data(ictx, this->m_test_data, TEST_IO_SIZE * i, + TEST_IO_SIZE); + } + this->close_image(ictx); + + sleep(DELAY / 2); + this->stop(); + this->start(); + + this->wait_for_replay_complete(); + delay = ceph_clock_now() - start_time; + ASSERT_GE(delay, DELAY); + + this->stop(); +} + +TYPED_TEST(TestImageReplayer, ImageRename) { + this->create_replayer(); + this->start(); + + librbd::ImageCtx* remote_image_ctx = nullptr; + this->open_remote_image(&remote_image_ctx); + auto image_name = this->get_temp_image_name(); + ASSERT_EQ(0, remote_image_ctx->operations->rename(image_name.c_str())); + this->flush(remote_image_ctx); + + this->wait_for_replay_complete(); + + librbd::ImageCtx* local_image_ctx = nullptr; + this->open_image(this->m_local_ioctx, image_name, true, &local_image_ctx); + ASSERT_EQ(image_name, local_image_ctx->name); + + this->close_image(local_image_ctx); + this->close_image(remote_image_ctx); + this->stop(); +} + +TYPED_TEST(TestImageReplayer, UpdateFeatures) { + const uint64_t FEATURES_TO_UPDATE = + RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF | RBD_FEATURE_DEEP_FLATTEN; + REQUIRE((this->FEATURES & FEATURES_TO_UPDATE) == FEATURES_TO_UPDATE); + + librbd::ImageCtx* remote_image_ctx = nullptr; + this->open_remote_image(&remote_image_ctx); + + ASSERT_EQ(0, remote_image_ctx->operations->update_features( + (RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF), false)); + this->flush(remote_image_ctx); + + this->create_replayer(); + this->start(); + this->wait_for_replay_complete(); + + librbd::ImageCtx* local_image_ctx = nullptr; + this->open_local_image(&local_image_ctx); + ASSERT_EQ(0U, local_image_ctx->features & ( + RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF)); + + // enable object-map/fast-diff + ASSERT_EQ(0, remote_image_ctx->operations->update_features( + (RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF), true)); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + ASSERT_EQ(0, local_image_ctx->state->refresh()); + ASSERT_EQ(RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF, + local_image_ctx->features & ( + RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF)); + + // disable deep-flatten + ASSERT_EQ(0, remote_image_ctx->operations->update_features( + RBD_FEATURE_DEEP_FLATTEN, false)); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + ASSERT_EQ(0, local_image_ctx->state->refresh()); + ASSERT_EQ(0, local_image_ctx->features & RBD_FEATURE_DEEP_FLATTEN); + + this->close_image(local_image_ctx); + this->close_image(remote_image_ctx); + this->stop(); +} + +TYPED_TEST(TestImageReplayer, SnapshotUnprotect) { + librbd::ImageCtx* remote_image_ctx = nullptr; + this->open_remote_image(&remote_image_ctx); + + // create a protected snapshot + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, remote_image_ctx->operations->snap_create( + cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); + ASSERT_EQ(0, remote_image_ctx->operations->snap_protect( + cls::rbd::UserSnapshotNamespace{}, "snap1")); + this->flush(remote_image_ctx); + + this->create_replayer(); + this->start(); + this->wait_for_replay_complete(); + + librbd::ImageCtx* local_image_ctx = nullptr; + this->open_local_image(&local_image_ctx); + auto local_snap_id_it = local_image_ctx->snap_ids.find({ + {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); + ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); + auto local_snap_id = local_snap_id_it->second; + auto local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); + ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); + ASSERT_EQ(RBD_PROTECTION_STATUS_PROTECTED, + local_snap_info_it->second.protection_status); + + // unprotect the snapshot + ASSERT_EQ(0, remote_image_ctx->operations->snap_unprotect( + cls::rbd::UserSnapshotNamespace{}, "snap1")); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + ASSERT_EQ(0, local_image_ctx->state->refresh()); + local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); + ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); + ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, + local_snap_info_it->second.protection_status); + + this->close_image(local_image_ctx); + this->close_image(remote_image_ctx); + this->stop(); +} + +TYPED_TEST(TestImageReplayer, SnapshotProtect) { + librbd::ImageCtx* remote_image_ctx = nullptr; + this->open_remote_image(&remote_image_ctx); + + // create an unprotected snapshot + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, remote_image_ctx->operations->snap_create( + cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); + this->flush(remote_image_ctx); + + this->create_replayer(); + this->start(); + this->wait_for_replay_complete(); + + librbd::ImageCtx* local_image_ctx = nullptr; + this->open_local_image(&local_image_ctx); + auto local_snap_id_it = local_image_ctx->snap_ids.find({ + {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); + ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); + auto local_snap_id = local_snap_id_it->second; + auto local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); + ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); + ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, + local_snap_info_it->second.protection_status); + + // protect the snapshot + ASSERT_EQ(0, remote_image_ctx->operations->snap_protect( + cls::rbd::UserSnapshotNamespace{}, "snap1")); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + ASSERT_EQ(0, local_image_ctx->state->refresh()); + local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); + ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); + ASSERT_EQ(RBD_PROTECTION_STATUS_PROTECTED, + local_snap_info_it->second.protection_status); + + this->close_image(local_image_ctx); + this->close_image(remote_image_ctx); + this->stop(); +} + +TYPED_TEST(TestImageReplayer, SnapshotRemove) { + librbd::ImageCtx* remote_image_ctx = nullptr; + this->open_remote_image(&remote_image_ctx); + + // create a user snapshot + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, remote_image_ctx->operations->snap_create( + cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); + this->flush(remote_image_ctx); + + this->create_replayer(); + this->start(); + this->wait_for_replay_complete(); + + librbd::ImageCtx* local_image_ctx = nullptr; + this->open_local_image(&local_image_ctx); + auto local_snap_id_it = local_image_ctx->snap_ids.find({ + {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); + ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); + + // remove the snapshot + ASSERT_EQ(0, remote_image_ctx->operations->snap_remove( + cls::rbd::UserSnapshotNamespace{}, "snap1")); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + ASSERT_EQ(0, local_image_ctx->state->refresh()); + local_snap_id_it = local_image_ctx->snap_ids.find({ + {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); + ASSERT_EQ(local_image_ctx->snap_ids.end(), local_snap_id_it); + + this->close_image(local_image_ctx); + this->close_image(remote_image_ctx); + this->stop(); +} + +TYPED_TEST(TestImageReplayer, SnapshotRename) { + librbd::ImageCtx* remote_image_ctx = nullptr; + this->open_remote_image(&remote_image_ctx); + + // create a user snapshot + librbd::NoOpProgressContext prog_ctx; + ASSERT_EQ(0, remote_image_ctx->operations->snap_create( + cls::rbd::UserSnapshotNamespace{}, "snap1", 0, prog_ctx)); + this->flush(remote_image_ctx); + + this->create_replayer(); + this->start(); + this->wait_for_replay_complete(); + + librbd::ImageCtx* local_image_ctx = nullptr; + this->open_local_image(&local_image_ctx); + auto local_snap_id_it = local_image_ctx->snap_ids.find({ + {cls::rbd::UserSnapshotNamespace{}}, "snap1"}); + ASSERT_NE(local_image_ctx->snap_ids.end(), local_snap_id_it); + auto local_snap_id = local_snap_id_it->second; + auto local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); + ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); + ASSERT_EQ(RBD_PROTECTION_STATUS_UNPROTECTED, + local_snap_info_it->second.protection_status); + + // rename the snapshot + ASSERT_EQ(0, remote_image_ctx->operations->snap_rename( + "snap1", "snap1-renamed")); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + ASSERT_EQ(0, local_image_ctx->state->refresh()); + local_snap_info_it = local_image_ctx->snap_info.find(local_snap_id); + ASSERT_NE(local_image_ctx->snap_info.end(), local_snap_info_it); + ASSERT_EQ("snap1-renamed", local_snap_info_it->second.name); + + this->close_image(local_image_ctx); + this->close_image(remote_image_ctx); + this->stop(); +} + +TYPED_TEST(TestImageReplayer, SnapshotLimit) { + librbd::ImageCtx* remote_image_ctx = nullptr; + this->open_remote_image(&remote_image_ctx); + + this->create_replayer(); + this->start(); + this->wait_for_replay_complete(); + + // update the snap limit + ASSERT_EQ(0, librbd::api::Snapshot<>::set_limit(remote_image_ctx, 123U)); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + librbd::ImageCtx* local_image_ctx = nullptr; + this->open_local_image(&local_image_ctx); + uint64_t local_snap_limit; + ASSERT_EQ(0, librbd::api::Snapshot<>::get_limit(local_image_ctx, + &local_snap_limit)); + ASSERT_EQ(123U, local_snap_limit); + + // update the limit again + ASSERT_EQ(0, librbd::api::Snapshot<>::set_limit( + remote_image_ctx, std::numeric_limits<uint64_t>::max())); + this->flush(remote_image_ctx); + this->wait_for_replay_complete(); + + ASSERT_EQ(0, librbd::api::Snapshot<>::get_limit(local_image_ctx, + &local_snap_limit)); + ASSERT_EQ(std::numeric_limits<uint64_t>::max(), local_snap_limit); + + this->close_image(local_image_ctx); + this->close_image(remote_image_ctx); + this->stop(); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_ImageSync.cc b/src/test/rbd_mirror/test_ImageSync.cc new file mode 100644 index 000000000..93349ca11 --- /dev/null +++ b/src/test/rbd_mirror/test_ImageSync.cc @@ -0,0 +1,374 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_fixture.h" +#include "include/stringify.h" +#include "include/rbd/librbd.hpp" +#include "common/Cond.h" +#include "journal/Journaler.h" +#include "journal/Settings.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/internal.h" +#include "librbd/Journal.h" +#include "librbd/Operations.h" +#include "librbd/api/Io.h" +#include "librbd/io/AioCompletion.h" +#include "librbd/io/ImageDispatchSpec.h" +#include "librbd/io/ReadResult.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/ImageSync.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Throttler.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" + +void register_test_image_sync() { +} + +namespace rbd { +namespace mirror { + +namespace { + +int flush(librbd::ImageCtx *image_ctx) { + C_SaferCond ctx; + auto aio_comp = librbd::io::AioCompletion::create_and_start( + &ctx, image_ctx, librbd::io::AIO_TYPE_FLUSH); + auto req = librbd::io::ImageDispatchSpec::create_flush( + *image_ctx, librbd::io::IMAGE_DISPATCH_LAYER_INTERNAL_START, aio_comp, + librbd::io::FLUSH_SOURCE_INTERNAL, {}); + req->send(); + return ctx.wait(); +} + +void scribble(librbd::ImageCtx *image_ctx, int num_ops, uint64_t max_size) +{ + max_size = std::min<uint64_t>(image_ctx->size, max_size); + for (int i=0; i<num_ops; i++) { + uint64_t off = rand() % (image_ctx->size - max_size + 1); + uint64_t len = 1 + rand() % max_size; + + if (rand() % 4 == 0) { + ASSERT_EQ((int)len, + librbd::api::Io<>::discard( + *image_ctx, off, len, image_ctx->discard_granularity_bytes)); + } else { + bufferlist bl; + bl.append(std::string(len, '1')); + ASSERT_EQ((int)len, librbd::api::Io<>::write( + *image_ctx, off, len, std::move(bl), 0)); + } + } + + std::shared_lock owner_locker{image_ctx->owner_lock}; + ASSERT_EQ(0, flush(image_ctx)); +} + +} // anonymous namespace +class TestImageSync : public TestFixture { +public: + + void SetUp() override { + TestFixture::SetUp(); + create_and_open(m_local_io_ctx, &m_local_image_ctx); + create_and_open(m_remote_io_ctx, &m_remote_image_ctx); + + auto cct = reinterpret_cast<CephContext*>(m_local_io_ctx.cct()); + m_image_sync_throttler = rbd::mirror::Throttler<>::create( + cct, "rbd_mirror_concurrent_image_syncs"); + + m_instance_watcher = rbd::mirror::InstanceWatcher<>::create( + m_local_io_ctx, *m_threads->asio_engine, nullptr, m_image_sync_throttler); + m_instance_watcher->handle_acquire_leader(); + + ContextWQ* context_wq; + librbd::Journal<>::get_work_queue(cct, &context_wq); + + m_remote_journaler = new ::journal::Journaler( + context_wq, m_threads->timer, &m_threads->timer_lock, + m_remote_io_ctx, m_remote_image_ctx->id, "mirror-uuid", {}, nullptr); + + m_client_meta = {"image-id"}; + + librbd::journal::ClientData client_data(m_client_meta); + bufferlist client_data_bl; + encode(client_data, client_data_bl); + + ASSERT_EQ(0, m_remote_journaler->register_client(client_data_bl)); + + m_state_builder = rbd::mirror::image_replayer::journal::StateBuilder< + librbd::ImageCtx>::create("global image id"); + m_state_builder->remote_journaler = m_remote_journaler; + m_state_builder->remote_client_meta = m_client_meta; + m_sync_point_handler = m_state_builder->create_sync_point_handler(); + } + + void TearDown() override { + m_instance_watcher->handle_release_leader(); + + m_state_builder->remote_journaler = nullptr; + m_state_builder->destroy_sync_point_handler(); + m_state_builder->destroy(); + + delete m_remote_journaler; + delete m_instance_watcher; + delete m_image_sync_throttler; + + TestFixture::TearDown(); + } + + void create_and_open(librados::IoCtx &io_ctx, librbd::ImageCtx **image_ctx) { + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(io_ctx, m_image_name, image_ctx)); + + C_SaferCond ctx; + { + std::shared_lock owner_locker{(*image_ctx)->owner_lock}; + (*image_ctx)->exclusive_lock->try_acquire_lock(&ctx); + } + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE((*image_ctx)->exclusive_lock->is_lock_owner()); + } + + ImageSync<> *create_request(Context *ctx) { + return new ImageSync<>(m_threads, m_local_image_ctx, m_remote_image_ctx, + "mirror-uuid", m_sync_point_handler, + m_instance_watcher, nullptr, ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx; + rbd::mirror::Throttler<> *m_image_sync_throttler; + rbd::mirror::InstanceWatcher<> *m_instance_watcher; + ::journal::Journaler *m_remote_journaler; + librbd::journal::MirrorPeerClientMeta m_client_meta; + rbd::mirror::image_replayer::journal::StateBuilder<librbd::ImageCtx>* m_state_builder = nullptr; + rbd::mirror::image_sync::SyncPointHandler* m_sync_point_handler = nullptr; +}; + +TEST_F(TestImageSync, Empty) { + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(0U, m_client_meta.sync_points.size()); + ASSERT_EQ(0, m_remote_image_ctx->state->refresh()); + ASSERT_EQ(0U, m_remote_image_ctx->snap_ids.size()); + ASSERT_EQ(0, m_local_image_ctx->state->refresh()); + ASSERT_EQ(1U, m_local_image_ctx->snap_ids.size()); // deleted on journal replay +} + +TEST_F(TestImageSync, Simple) { + scribble(m_remote_image_ctx, 10, 102400); + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + bufferlist read_remote_bl; + read_remote_bl.append(std::string(object_size, '1')); + bufferlist read_local_bl; + read_local_bl.append(std::string(object_size, '1')); + + for (uint64_t offset = 0; offset < m_remote_image_ctx->size; + offset += object_size) { + ASSERT_LE(0, librbd::api::Io<>::read( + *m_remote_image_ctx, offset, object_size, + librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, librbd::api::Io<>::read( + *m_local_image_ctx, offset, object_size, + librbd::io::ReadResult{&read_local_bl}, 0)); + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); + } +} + +TEST_F(TestImageSync, Resize) { + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + + uint64_t off = 0; + uint64_t len = object_size / 10; + + bufferlist bl; + bl.append(std::string(len, '1')); + ASSERT_EQ((int)len, librbd::api::Io<>::write( + *m_remote_image_ctx, off, len, std::move(bl), 0)); + { + std::shared_lock owner_locker{m_remote_image_ctx->owner_lock}; + ASSERT_EQ(0, flush(m_remote_image_ctx)); + } + + ASSERT_EQ(0, create_snap(m_remote_image_ctx, "snap", nullptr)); + + uint64_t size = object_size - 1; + librbd::NoOpProgressContext no_op_progress_ctx; + ASSERT_EQ(0, m_remote_image_ctx->operations->resize(size, true, + no_op_progress_ctx)); + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + bufferlist read_remote_bl; + read_remote_bl.append(std::string(len, '\0')); + bufferlist read_local_bl; + read_local_bl.append(std::string(len, '\0')); + + ASSERT_LE(0, librbd::api::Io<>::read( + *m_remote_image_ctx, off, len, + librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, librbd::api::Io<>::read( + *m_local_image_ctx, off, len, + librbd::io::ReadResult{&read_local_bl}, 0)); + + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); +} + +TEST_F(TestImageSync, Discard) { + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + + uint64_t off = 0; + uint64_t len = object_size / 10; + + bufferlist bl; + bl.append(std::string(len, '1')); + ASSERT_EQ((int)len, librbd::api::Io<>::write( + *m_remote_image_ctx, off, len, std::move(bl), 0)); + { + std::shared_lock owner_locker{m_remote_image_ctx->owner_lock}; + ASSERT_EQ(0, flush(m_remote_image_ctx)); + } + + ASSERT_EQ(0, create_snap(m_remote_image_ctx, "snap", nullptr)); + + ASSERT_EQ((int)len - 2, + librbd::api::Io<>::discard( + *m_remote_image_ctx, off + 1, len - 2, + m_remote_image_ctx->discard_granularity_bytes)); + { + std::shared_lock owner_locker{m_remote_image_ctx->owner_lock}; + ASSERT_EQ(0, flush(m_remote_image_ctx)); + } + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + bufferlist read_remote_bl; + read_remote_bl.append(std::string(object_size, '\0')); + bufferlist read_local_bl; + read_local_bl.append(std::string(object_size, '\0')); + + ASSERT_LE(0, librbd::api::Io<>::read( + *m_remote_image_ctx, off, len, + librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, librbd::api::Io<>::read( + *m_local_image_ctx, off, len, + librbd::io::ReadResult{&read_local_bl}, 0)); + + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); +} + +TEST_F(TestImageSync, SnapshotStress) { + std::list<std::string> snap_names; + + const int num_snaps = 4; + for (int idx = 0; idx <= num_snaps; ++idx) { + scribble(m_remote_image_ctx, 10, 102400); + + librbd::NoOpProgressContext no_op_progress_ctx; + uint64_t size = 1 + rand() % m_image_size; + ASSERT_EQ(0, m_remote_image_ctx->operations->resize(size, true, + no_op_progress_ctx)); + ASSERT_EQ(0, m_remote_image_ctx->state->refresh()); + + if (idx < num_snaps) { + snap_names.push_back("snap" + stringify(idx + 1)); + ASSERT_EQ(0, create_snap(m_remote_image_ctx, snap_names.back().c_str(), + nullptr)); + } else { + snap_names.push_back(""); + } + } + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + bufferlist read_remote_bl; + read_remote_bl.append(std::string(object_size, '1')); + bufferlist read_local_bl; + read_local_bl.append(std::string(object_size, '1')); + + for (auto &snap_name : snap_names) { + uint64_t remote_snap_id; + { + std::shared_lock remote_image_locker{m_remote_image_ctx->image_lock}; + remote_snap_id = m_remote_image_ctx->get_snap_id( + cls::rbd::UserSnapshotNamespace{}, snap_name); + } + + uint64_t remote_size; + { + C_SaferCond ctx; + m_remote_image_ctx->state->snap_set(remote_snap_id, &ctx); + ASSERT_EQ(0, ctx.wait()); + + std::shared_lock remote_image_locker{m_remote_image_ctx->image_lock}; + remote_size = m_remote_image_ctx->get_image_size( + m_remote_image_ctx->snap_id); + } + + uint64_t local_snap_id; + { + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + local_snap_id = m_local_image_ctx->get_snap_id( + cls::rbd::UserSnapshotNamespace{}, snap_name); + } + + uint64_t local_size; + { + C_SaferCond ctx; + m_local_image_ctx->state->snap_set(local_snap_id, &ctx); + ASSERT_EQ(0, ctx.wait()); + + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + local_size = m_local_image_ctx->get_image_size( + m_local_image_ctx->snap_id); + bool flags_set; + ASSERT_EQ(0, m_local_image_ctx->test_flags(m_local_image_ctx->snap_id, + RBD_FLAG_OBJECT_MAP_INVALID, + m_local_image_ctx->image_lock, + &flags_set)); + ASSERT_FALSE(flags_set); + } + + ASSERT_EQ(remote_size, local_size); + + for (uint64_t offset = 0; offset < remote_size; offset += object_size) { + ASSERT_LE(0, librbd::api::Io<>::read( + *m_remote_image_ctx, offset, object_size, + librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, librbd::api::Io<>::read( + *m_local_image_ctx, offset, object_size, + librbd::io::ReadResult{&read_local_bl}, 0)); + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); + } + } +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_InstanceWatcher.cc b/src/test/rbd_mirror/test_InstanceWatcher.cc new file mode 100644 index 000000000..6b8176d8a --- /dev/null +++ b/src/test/rbd_mirror/test_InstanceWatcher.cc @@ -0,0 +1,132 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "include/stringify.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/Utils.h" +#include "librbd/internal.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "common/Cond.h" + +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" + +using rbd::mirror::InstanceWatcher; + +void register_test_instance_watcher() { +} + +class TestInstanceWatcher : public ::rbd::mirror::TestFixture { +public: + std::string m_instance_id; + std::string m_oid; + + void SetUp() override { + TestFixture::SetUp(); + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); + + m_instance_id = stringify(m_local_io_ctx.get_instance_id()); + m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id; + } + + void get_instances(std::vector<std::string> *instance_ids) { + instance_ids->clear(); + C_SaferCond on_get; + InstanceWatcher<>::get_instances(m_local_io_ctx, instance_ids, &on_get); + EXPECT_EQ(0, on_get.wait()); + } +}; + +TEST_F(TestInstanceWatcher, InitShutdown) +{ + InstanceWatcher<> instance_watcher(m_local_io_ctx, *m_threads->asio_engine, + nullptr, nullptr, m_instance_id); + std::vector<std::string> instance_ids; + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + uint64_t size; + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr)); + + // Init + ASSERT_EQ(0, instance_watcher.init()); + + get_instances(&instance_ids); + ASSERT_EQ(1U, instance_ids.size()); + ASSERT_EQ(m_instance_id, instance_ids[0]); + + ASSERT_EQ(0, m_local_io_ctx.stat(m_oid, &size, nullptr)); + std::list<obj_watch_t> watchers; + ASSERT_EQ(0, m_local_io_ctx.list_watchers(m_oid, &watchers)); + ASSERT_EQ(1U, watchers.size()); + ASSERT_EQ(m_instance_id, stringify(watchers.begin()->watcher_id)); + + get_instances(&instance_ids); + ASSERT_EQ(1U, instance_ids.size()); + + // Shutdown + instance_watcher.shut_down(); + + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr)); + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); +} + +TEST_F(TestInstanceWatcher, Remove) +{ + std::string instance_id = "instance_id"; + std::string oid = RBD_MIRROR_INSTANCE_PREFIX + instance_id; + + std::vector<std::string> instance_ids; + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + uint64_t size; + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(oid, &size, nullptr)); + + librados::Rados cluster; + librados::IoCtx io_ctx; + ASSERT_EQ("", connect_cluster_pp(cluster)); + ASSERT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx)); + InstanceWatcher<> instance_watcher(m_local_io_ctx, *m_threads->asio_engine, + nullptr, nullptr, "instance_id"); + // Init + ASSERT_EQ(0, instance_watcher.init()); + + get_instances(&instance_ids); + ASSERT_EQ(1U, instance_ids.size()); + ASSERT_EQ(instance_id, instance_ids[0]); + + ASSERT_EQ(0, m_local_io_ctx.stat(oid, &size, nullptr)); + std::list<obj_watch_t> watchers; + ASSERT_EQ(0, m_local_io_ctx.list_watchers(oid, &watchers)); + ASSERT_EQ(1U, watchers.size()); + + // Remove + C_SaferCond on_remove; + InstanceWatcher<>::remove_instance(m_local_io_ctx, *m_threads->asio_engine, + "instance_id", &on_remove); + ASSERT_EQ(0, on_remove.wait()); + + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(oid, &size, nullptr)); + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + // Shutdown + instance_watcher.shut_down(); + + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr)); + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + // Remove NOENT + C_SaferCond on_remove_noent; + InstanceWatcher<>::remove_instance(m_local_io_ctx, *m_threads->asio_engine, + instance_id, &on_remove_noent); + ASSERT_EQ(0, on_remove_noent.wait()); +} diff --git a/src/test/rbd_mirror/test_Instances.cc b/src/test/rbd_mirror/test_Instances.cc new file mode 100644 index 000000000..4b189d903 --- /dev/null +++ b/src/test/rbd_mirror/test_Instances.cc @@ -0,0 +1,164 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_client.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Instances.h" +#include "tools/rbd_mirror/Threads.h" +#include "common/Cond.h" + +#include "test/librados/test.h" +#include "gtest/gtest.h" +#include <vector> + +using rbd::mirror::InstanceWatcher; +using rbd::mirror::Instances; + +void register_test_instances() { +} + +class TestInstances : public ::rbd::mirror::TestFixture { +public: + struct Listener : public rbd::mirror::instances::Listener { + std::mutex lock; + + struct Instance { + uint32_t count = 0; + std::set<std::string> ids; + C_SaferCond ctx; + }; + + Instance add; + Instance remove; + + void handle(const InstanceIds& instance_ids, Instance* instance) { + std::unique_lock<std::mutex> locker(lock); + for (auto& instance_id : instance_ids) { + ceph_assert(instance->count > 0); + --instance->count; + + instance->ids.insert(instance_id); + if (instance->count == 0) { + instance->ctx.complete(0); + } + } + } + + void handle_added(const InstanceIds& instance_ids) override { + handle(instance_ids, &add); + } + + void handle_removed(const InstanceIds& instance_ids) override { + handle(instance_ids, &remove); + } + }; + + virtual void SetUp() { + TestFixture::SetUp(); + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); + + m_instance_id = stringify(m_local_io_ctx.get_instance_id()); + } + + Listener m_listener; + std::string m_instance_id; +}; + +TEST_F(TestInstances, InitShutdown) +{ + m_listener.add.count = 1; + Instances<> instances(m_threads, m_local_io_ctx, m_instance_id, m_listener); + + std::string instance_id = "instance_id"; + ASSERT_EQ(0, librbd::cls_client::mirror_instances_add(&m_local_io_ctx, + instance_id)); + + C_SaferCond on_init; + instances.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + ASSERT_LT(0U, m_listener.add.count); + instances.unblock_listener(); + + ASSERT_EQ(0, m_listener.add.ctx.wait()); + ASSERT_EQ(std::set<std::string>({instance_id}), m_listener.add.ids); + + C_SaferCond on_shut_down; + instances.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} + +TEST_F(TestInstances, InitEnoent) +{ + Instances<> instances(m_threads, m_local_io_ctx, m_instance_id, m_listener); + + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + + C_SaferCond on_init; + instances.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + C_SaferCond on_shut_down; + instances.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} + +TEST_F(TestInstances, NotifyRemove) +{ + // speed testing up a little + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", "1")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_missed_heartbeats", + "2")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_acquire_attempts_before_break", + "0")); + + m_listener.add.count = 2; + m_listener.remove.count = 1; + Instances<> instances(m_threads, m_local_io_ctx, m_instance_id, m_listener); + + std::string instance_id1 = "instance_id1"; + std::string instance_id2 = "instance_id2"; + + ASSERT_EQ(0, librbd::cls_client::mirror_instances_add(&m_local_io_ctx, + instance_id1)); + + C_SaferCond on_init; + instances.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + instances.acked({instance_id2}); + + ASSERT_LT(0U, m_listener.add.count); + instances.unblock_listener(); + + ASSERT_EQ(0, m_listener.add.ctx.wait()); + ASSERT_EQ(std::set<std::string>({instance_id1, instance_id2}), + m_listener.add.ids); + + std::vector<std::string> instance_ids; + for (int i = 0; i < 100; i++) { + instances.acked({instance_id1}); + if (m_listener.remove.count > 0) { + usleep(250000); + } + } + + instances.acked({instance_id1}); + ASSERT_EQ(0, m_listener.remove.ctx.wait()); + ASSERT_EQ(std::set<std::string>({instance_id2}), + m_listener.remove.ids); + + C_SaferCond on_get; + instances.acked({instance_id1}); + InstanceWatcher<>::get_instances(m_local_io_ctx, &instance_ids, &on_get); + EXPECT_EQ(0, on_get.wait()); + EXPECT_EQ(1U, instance_ids.size()); + ASSERT_EQ(instance_ids[0], instance_id1); + + C_SaferCond on_shut_down; + instances.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} diff --git a/src/test/rbd_mirror/test_LeaderWatcher.cc b/src/test/rbd_mirror/test_LeaderWatcher.cc new file mode 100644 index 000000000..ac8f1bb0a --- /dev/null +++ b/src/test/rbd_mirror/test_LeaderWatcher.cc @@ -0,0 +1,318 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "librbd/internal.h" +#include "librbd/Utils.h" +#include "librbd/api/Mirror.h" +#include "test/librbd/test_support.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/LeaderWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "common/Cond.h" + +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" + +using librbd::util::unique_lock_name; +using rbd::mirror::LeaderWatcher; + +void register_test_leader_watcher() { +} + +class TestLeaderWatcher : public ::rbd::mirror::TestFixture { +public: + class Listener : public rbd::mirror::leader_watcher::Listener { + public: + Listener() + : m_test_lock(ceph::make_mutex( + unique_lock_name("LeaderWatcher::m_test_lock", this))) { + } + + void on_acquire(int r, Context *ctx) { + std::lock_guard locker{m_test_lock}; + m_on_acquire_r = r; + m_on_acquire = ctx; + } + + void on_release(int r, Context *ctx) { + std::lock_guard locker{m_test_lock}; + m_on_release_r = r; + m_on_release = ctx; + } + + int acquire_count() const { + std::lock_guard locker{m_test_lock}; + return m_acquire_count; + } + + int release_count() const { + std::lock_guard locker{m_test_lock}; + return m_release_count; + } + + void post_acquire_handler(Context *on_finish) override { + std::lock_guard locker{m_test_lock}; + m_acquire_count++; + on_finish->complete(m_on_acquire_r); + m_on_acquire_r = 0; + if (m_on_acquire != nullptr) { + m_on_acquire->complete(0); + m_on_acquire = nullptr; + } + } + + void pre_release_handler(Context *on_finish) override { + std::lock_guard locker{m_test_lock}; + m_release_count++; + on_finish->complete(m_on_release_r); + m_on_release_r = 0; + if (m_on_release != nullptr) { + m_on_release->complete(0); + m_on_release = nullptr; + } + } + + void update_leader_handler(const std::string &leader_instance_id) override { + } + + void handle_instances_added(const InstanceIds& instance_ids) override { + } + void handle_instances_removed(const InstanceIds& instance_ids) override { + } + + private: + mutable ceph::mutex m_test_lock; + int m_acquire_count = 0; + int m_release_count = 0; + int m_on_acquire_r = 0; + int m_on_release_r = 0; + Context *m_on_acquire = nullptr; + Context *m_on_release = nullptr; + }; + + struct Connection { + librados::Rados cluster; + librados::IoCtx io_ctx; + }; + + std::list<std::unique_ptr<Connection> > m_connections; + + void SetUp() override { + TestFixture::SetUp(); + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_local_io_ctx, + RBD_MIRROR_MODE_POOL)); + + if (is_librados_test_stub(*_rados)) { + // speed testing up a little + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", + "1")); + } + } + + librados::IoCtx &create_connection(bool no_heartbeats = false) { + m_connections.push_back(std::unique_ptr<Connection>(new Connection())); + Connection *c = m_connections.back().get(); + + EXPECT_EQ("", connect_cluster_pp(c->cluster)); + if (no_heartbeats) { + EXPECT_EQ(0, c->cluster.conf_set("rbd_mirror_leader_heartbeat_interval", + "3600")); + } else if (is_librados_test_stub(*_rados)) { + EXPECT_EQ(0, c->cluster.conf_set("rbd_mirror_leader_heartbeat_interval", + "1")); + } + EXPECT_EQ(0, c->cluster.ioctx_create(_local_pool_name.c_str(), c->io_ctx)); + + return c->io_ctx; + } +}; + +TEST_F(TestLeaderWatcher, InitShutdown) +{ + Listener listener; + LeaderWatcher<> leader_watcher(m_threads, m_local_io_ctx, &listener); + + C_SaferCond on_init_acquire; + listener.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + leader_watcher.shut_down(); + ASSERT_EQ(1, listener.acquire_count()); + ASSERT_EQ(1, listener.release_count()); + ASSERT_FALSE(leader_watcher.is_leader()); +} + +TEST_F(TestLeaderWatcher, Release) +{ + Listener listener; + LeaderWatcher<> leader_watcher(m_threads, m_local_io_ctx, &listener); + + C_SaferCond on_init_acquire; + listener.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + C_SaferCond on_release; + C_SaferCond on_acquire; + listener.on_release(0, &on_release); + listener.on_acquire(0, &on_acquire); + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release.wait()); + ASSERT_FALSE(leader_watcher.is_leader()); + + // wait for lock re-acquired due to no another locker + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + C_SaferCond on_release2; + listener.on_release(0, &on_release2); + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release2.wait()); + + leader_watcher.shut_down(); + ASSERT_EQ(2, listener.acquire_count()); + ASSERT_EQ(2, listener.release_count()); +} + +TEST_F(TestLeaderWatcher, ListenerError) +{ + Listener listener; + LeaderWatcher<> leader_watcher(m_threads, m_local_io_ctx, &listener); + + // make listener return error on acquire + C_SaferCond on_init_acquire, on_init_release; + listener.on_acquire(-EINVAL, &on_init_acquire); + listener.on_release(0, &on_init_release); + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + ASSERT_EQ(0, on_init_release.wait()); + ASSERT_FALSE(leader_watcher.is_leader()); + + // wait for lock re-acquired due to no another locker + C_SaferCond on_acquire; + listener.on_acquire(0, &on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + // make listener return error on release + C_SaferCond on_release; + listener.on_release(-EINVAL, &on_release); + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release.wait()); + ASSERT_FALSE(leader_watcher.is_leader()); + + leader_watcher.shut_down(); + ASSERT_EQ(2, listener.acquire_count()); + ASSERT_EQ(2, listener.release_count()); + ASSERT_FALSE(leader_watcher.is_leader()); +} + +TEST_F(TestLeaderWatcher, Two) +{ + Listener listener1; + LeaderWatcher<> leader_watcher1(m_threads, create_connection(), &listener1); + + C_SaferCond on_init_acquire; + listener1.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher1.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + + Listener listener2; + LeaderWatcher<> leader_watcher2(m_threads, create_connection(), &listener2); + + ASSERT_EQ(0, leader_watcher2.init()); + ASSERT_TRUE(leader_watcher1.is_leader()); + ASSERT_FALSE(leader_watcher2.is_leader()); + + C_SaferCond on_release; + C_SaferCond on_acquire; + listener1.on_release(0, &on_release); + listener2.on_acquire(0, &on_acquire); + leader_watcher1.release_leader(); + ASSERT_EQ(0, on_release.wait()); + ASSERT_FALSE(leader_watcher1.is_leader()); + + // wait for lock acquired by another watcher + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher2.is_leader()); + + leader_watcher1.shut_down(); + leader_watcher2.shut_down(); + + ASSERT_EQ(1, listener1.acquire_count()); + ASSERT_EQ(1, listener1.release_count()); + ASSERT_EQ(1, listener2.acquire_count()); + ASSERT_EQ(1, listener2.release_count()); +} + +TEST_F(TestLeaderWatcher, Break) +{ + Listener listener1, listener2; + LeaderWatcher<> leader_watcher1(m_threads, + create_connection(true /* no heartbeats */), + &listener1); + LeaderWatcher<> leader_watcher2(m_threads, create_connection(), &listener2); + + C_SaferCond on_init_acquire; + listener1.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher1.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + + C_SaferCond on_acquire; + listener2.on_acquire(0, &on_acquire); + ASSERT_EQ(0, leader_watcher2.init()); + ASSERT_FALSE(leader_watcher2.is_leader()); + + // wait for lock broken due to no heartbeats and re-acquired + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher2.is_leader()); + + leader_watcher1.shut_down(); + leader_watcher2.shut_down(); +} + +TEST_F(TestLeaderWatcher, Stress) +{ + const int WATCHERS_COUNT = 20; + std::list<LeaderWatcher<> *> leader_watchers; + Listener listener; + + for (int i = 0; i < WATCHERS_COUNT; i++) { + auto leader_watcher = + new LeaderWatcher<>(m_threads, create_connection(), &listener); + leader_watchers.push_back(leader_watcher); + } + + C_SaferCond on_init_acquire; + listener.on_acquire(0, &on_init_acquire); + for (auto &leader_watcher : leader_watchers) { + ASSERT_EQ(0, leader_watcher->init()); + } + ASSERT_EQ(0, on_init_acquire.wait()); + + while (true) { + C_SaferCond on_acquire; + listener.on_acquire(0, &on_acquire); + std::unique_ptr<LeaderWatcher<> > leader_watcher; + for (auto it = leader_watchers.begin(); it != leader_watchers.end(); ) { + if ((*it)->is_leader()) { + ASSERT_FALSE(leader_watcher); + leader_watcher.reset(*it); + it = leader_watchers.erase(it); + } else { + it++; + } + } + + ASSERT_TRUE(leader_watcher); + leader_watcher->shut_down(); + if (leader_watchers.empty()) { + break; + } + ASSERT_EQ(0, on_acquire.wait()); + } +} diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc new file mode 100644 index 000000000..c29ed65af --- /dev/null +++ b/src/test/rbd_mirror/test_PoolWatcher.cc @@ -0,0 +1,254 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "include/stringify.h" +#include "test/rbd_mirror/test_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "include/rbd_types.h" +#include "librbd/internal.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/api/Mirror.h" +#include "common/Cond.h" +#include "common/errno.h" +#include "common/ceph_mutex.h" +#include "tools/rbd_mirror/PoolWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" +#include "tools/rbd_mirror/pool_watcher/Types.h" +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" +#include <boost/scope_exit.hpp> +#include <iostream> +#include <map> +#include <memory> +#include <set> +#include <vector> + +using rbd::mirror::ImageId; +using rbd::mirror::ImageIds; +using rbd::mirror::PoolWatcher; +using rbd::mirror::PeerSpec; +using rbd::mirror::RadosRef; +using std::map; +using std::set; +using std::string; + +void register_test_pool_watcher() { +} + +class TestPoolWatcher : public ::rbd::mirror::TestFixture { +public: + + TestPoolWatcher() + : m_pool_watcher_listener(this), + m_image_number(0), m_snap_number(0) + { + m_cluster = std::make_shared<librados::Rados>(); + EXPECT_EQ("", connect_cluster_pp(*m_cluster)); + } + + void TearDown() override { + if (m_pool_watcher) { + C_SaferCond ctx; + m_pool_watcher->shut_down(&ctx); + EXPECT_EQ(0, ctx.wait()); + } + + m_cluster->wait_for_latest_osdmap(); + for (auto& pool : m_pools) { + EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str())); + } + + TestFixture::TearDown(); + } + + struct PoolWatcherListener : public rbd::mirror::pool_watcher::Listener { + TestPoolWatcher *test; + ceph::condition_variable cond; + ImageIds image_ids; + + explicit PoolWatcherListener(TestPoolWatcher *test) : test(test) { + } + + void handle_update(const std::string &mirror_uuid, + ImageIds &&added_image_ids, + ImageIds &&removed_image_ids) override { + std::lock_guard locker{test->m_lock}; + for (auto &image_id : removed_image_ids) { + image_ids.erase(image_id); + } + image_ids.insert(added_image_ids.begin(), added_image_ids.end()); + cond.notify_all(); + } + }; + + void create_pool(bool enable_mirroring, const PeerSpec &peer, string *name=nullptr) { + string pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str())); + + int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); + ASSERT_GE(pool_id, 0); + m_pools.insert(pool_name); + + librados::IoCtx ioctx; + ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); + ioctx.application_enable("rbd", true); + + m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, "mirror uuid", + m_pool_watcher_listener)); + + if (enable_mirroring) { + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, + RBD_MIRROR_MODE_POOL)); + std::string uuid; + ASSERT_EQ(0, librbd::api::Mirror<>::peer_site_add( + ioctx, &uuid, RBD_MIRROR_PEER_DIRECTION_RX_TX, peer.cluster_name, + peer.client_name)); + } + if (name != nullptr) { + *name = pool_name; + } + + m_pool_watcher->init(); + } + + string get_image_id(librados::IoCtx *ioctx, const string &image_name) { + string obj = librbd::util::id_obj_name(image_name); + string id; + EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id)); + return id; + } + + void create_image(const string &pool_name, bool mirrored=true, + string *image_name=nullptr) { + uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); + string name = "image" + stringify(++m_image_number); + if (mirrored) { + features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; + } + + librados::IoCtx ioctx; + ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx)); + int order = 0; + ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false, + features, &order, 0, 0)); + if (mirrored) { + librbd::Image image; + librbd::RBD rbd; + rbd.open(ioctx, image, name.c_str()); + image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL); + + librbd::mirror_image_info_t mirror_image_info; + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, + sizeof(mirror_image_info))); + image.close(); + + m_mirrored_images.insert(ImageId( + mirror_image_info.global_id, get_image_id(&ioctx, name))); + } + if (image_name != nullptr) + *image_name = name; + } + + void clone_image(const string &parent_pool_name, + const string &parent_image_name, + const string &clone_pool_name, + bool mirrored=true, + string *image_name=nullptr) { + librados::IoCtx pioctx, cioctx; + ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx)); + ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx)); + + string snap_name = "snap" + stringify(++m_snap_number); + { + librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(), + "", "", pioctx, false); + ictx->state->open(0); + librbd::NoOpProgressContext prog_ctx; + EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + snap_name, 0, prog_ctx)); + EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + snap_name)); + ictx->state->close(); + } + + uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); + string name = "clone" + stringify(++m_image_number); + if (mirrored) { + features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; + } + int order = 0; + librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(), + cioctx, name.c_str(), features, &order, 0, 0); + if (mirrored) { + librbd::Image image; + librbd::RBD rbd; + rbd.open(cioctx, image, name.c_str()); + image.mirror_image_enable2(RBD_MIRROR_IMAGE_MODE_JOURNAL); + + librbd::mirror_image_info_t mirror_image_info; + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, + sizeof(mirror_image_info))); + image.close(); + + m_mirrored_images.insert(ImageId( + mirror_image_info.global_id, get_image_id(&cioctx, name))); + } + if (image_name != nullptr) + *image_name = name; + } + + void check_images() { + std::unique_lock l{m_lock}; + while (m_mirrored_images != m_pool_watcher_listener.image_ids) { + if (m_pool_watcher_listener.cond.wait_for(l, 10s) == std::cv_status::timeout) { + break; + } + } + + ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids); + } + + ceph::mutex m_lock = ceph::make_mutex("TestPoolWatcherLock"); + RadosRef m_cluster; + PoolWatcherListener m_pool_watcher_listener; + unique_ptr<PoolWatcher<> > m_pool_watcher; + + set<string> m_pools; + ImageIds m_mirrored_images; + + uint64_t m_image_number; + uint64_t m_snap_number; +}; + +TEST_F(TestPoolWatcher, EmptyPool) { + string uuid1 = "00000000-0000-0000-0000-000000000001"; + PeerSpec site1(uuid1, "site1", "mirror1"); + create_pool(true, site1); + check_images(); +} + +TEST_F(TestPoolWatcher, ReplicatedPools) { + string uuid1 = "00000000-0000-0000-0000-000000000001"; + PeerSpec site1(uuid1, "site1", "mirror1"); + string first_pool, local_pool, last_pool; + create_pool(true, site1, &first_pool); + check_images(); + create_image(first_pool); + check_images(); + string parent_image, parent_image2; + create_image(first_pool, true, &parent_image); + check_images(); + clone_image(first_pool, parent_image, first_pool); + check_images(); + clone_image(first_pool, parent_image, first_pool, true, &parent_image2); + check_images(); + create_image(first_pool, false); + check_images(); +} diff --git a/src/test/rbd_mirror/test_fixture.cc b/src/test/rbd_mirror/test_fixture.cc new file mode 100644 index 000000000..7abf0e3e2 --- /dev/null +++ b/src/test/rbd_mirror/test_fixture.cc @@ -0,0 +1,161 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "cls/rbd/cls_rbd_types.h" +#include "test/rbd_mirror/test_fixture.h" +#include "include/stringify.h" +#include "include/rbd/librbd.hpp" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/internal.h" +#include "test/librados/test_cxx.h" +#include "tools/rbd_mirror/Threads.h" + +namespace rbd { +namespace mirror { + +std::string TestFixture::_local_pool_name; +std::string TestFixture::_remote_pool_name; +std::shared_ptr<librados::Rados> TestFixture::_rados; +uint64_t TestFixture::_image_number = 0; +std::string TestFixture::_data_pool; + +TestFixture::TestFixture() { +} + +void TestFixture::SetUpTestCase() { + _rados = std::shared_ptr<librados::Rados>(new librados::Rados()); + ASSERT_EQ("", connect_cluster_pp(*_rados.get())); + ASSERT_EQ(0, _rados->conf_set("rbd_cache", "false")); + + _local_pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, _rados->pool_create(_local_pool_name.c_str())); + + librados::IoCtx local_ioctx; + ASSERT_EQ(0, _rados->ioctx_create(_local_pool_name.c_str(), local_ioctx)); + local_ioctx.application_enable("rbd", true); + + _remote_pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, _rados->pool_create(_remote_pool_name.c_str())); + + librados::IoCtx remote_ioctx; + ASSERT_EQ(0, _rados->ioctx_create(_remote_pool_name.c_str(), remote_ioctx)); + remote_ioctx.application_enable("rbd", true); + + ASSERT_EQ(0, create_image_data_pool(_data_pool)); + if (!_data_pool.empty()) { + printf("using image data pool: %s\n", _data_pool.c_str()); + } +} + +void TestFixture::TearDownTestCase() { + if (!_data_pool.empty()) { + ASSERT_EQ(0, _rados->pool_delete(_data_pool.c_str())); + } + + ASSERT_EQ(0, _rados->pool_delete(_remote_pool_name.c_str())); + ASSERT_EQ(0, _rados->pool_delete(_local_pool_name.c_str())); + _rados->shutdown(); +} + +void TestFixture::SetUp() { + static bool seeded = false; + if (!seeded) { + seeded = true; + int seed = getpid(); + cout << "seed " << seed << std::endl; + srand(seed); + } + + ASSERT_EQ(0, _rados->ioctx_create(_local_pool_name.c_str(), m_local_io_ctx)); + ASSERT_EQ(0, _rados->ioctx_create(_remote_pool_name.c_str(), m_remote_io_ctx)); + m_image_name = get_temp_image_name(); + + m_threads = new rbd::mirror::Threads<>(_rados); +} + +void TestFixture::TearDown() { + for (auto image_ctx : m_image_ctxs) { + image_ctx->state->close(); + } + + m_remote_io_ctx.close(); + m_local_io_ctx.close(); + + delete m_threads; +} + +int TestFixture::create_image(librbd::RBD &rbd, librados::IoCtx &ioctx, + const std::string &name, uint64_t size) { + int order = 18; + return rbd.create2(ioctx, name.c_str(), size, RBD_FEATURES_ALL, &order); +} + +int TestFixture::open_image(librados::IoCtx &io_ctx, + const std::string &image_name, + librbd::ImageCtx **image_ctx) { + *image_ctx = new librbd::ImageCtx(image_name.c_str(), "", nullptr, io_ctx, + false); + m_image_ctxs.insert(*image_ctx); + return (*image_ctx)->state->open(0); +} + +int TestFixture::create_snap(librbd::ImageCtx *image_ctx, const char* snap_name, + librados::snap_t *snap_id) { + librbd::NoOpProgressContext prog_ctx; + int r = image_ctx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + snap_name, 0, prog_ctx); + if (r < 0) { + return r; + } + + r = image_ctx->state->refresh(); + if (r < 0) { + return r; + } + + if (image_ctx->snap_ids.count({cls::rbd::UserSnapshotNamespace(), + snap_name}) == 0) { + return -ENOENT; + } + + if (snap_id != nullptr) { + *snap_id = image_ctx->snap_ids[{cls::rbd::UserSnapshotNamespace(), + snap_name}]; + } + return 0; +} + +std::string TestFixture::get_temp_image_name() { + ++_image_number; + return "image" + stringify(_image_number); +} + +int TestFixture::create_image_data_pool(std::string &data_pool) { + std::string pool; + int r = _rados->conf_get("rbd_default_data_pool", pool); + if (r != 0) { + return r; + } else if (pool.empty()) { + return 0; + } + + r = _rados->pool_create(pool.c_str()); + if (r < 0) { + return r; + } + + librados::IoCtx data_ioctx; + r = _rados->ioctx_create(pool.c_str(), data_ioctx); + if (r < 0) { + return r; + } + + data_ioctx.application_enable("rbd", true); + data_pool = pool; + return 0; +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_fixture.h b/src/test/rbd_mirror/test_fixture.h new file mode 100644 index 000000000..217ae8102 --- /dev/null +++ b/src/test/rbd_mirror/test_fixture.h @@ -0,0 +1,65 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_TEST_RBD_MIRROR_TEST_FIXTURE_H +#define CEPH_TEST_RBD_MIRROR_TEST_FIXTURE_H + +#include "include/int_types.h" +#include "include/rados/librados.hpp" +#include <gtest/gtest.h> +#include <memory> +#include <set> + +namespace librbd { +class ImageCtx; +class RBD; +} + +namespace rbd { +namespace mirror { + +template <typename> class Threads; + +class TestFixture : public ::testing::Test { +public: + TestFixture(); + + static void SetUpTestCase(); + static void TearDownTestCase(); + + void SetUp() override; + void TearDown() override; + + librados::IoCtx m_local_io_ctx; + librados::IoCtx m_remote_io_ctx; + + std::string m_image_name; + uint64_t m_image_size = 1 << 24; + + std::set<librbd::ImageCtx *> m_image_ctxs; + + Threads<librbd::ImageCtx> *m_threads = nullptr; + + + int create_image(librbd::RBD &rbd, librados::IoCtx &ioctx, + const std::string &name, uint64_t size); + int open_image(librados::IoCtx &io_ctx, const std::string &image_name, + librbd::ImageCtx **image_ctx); + + int create_snap(librbd::ImageCtx *image_ctx, const char* snap_name, + librados::snap_t *snap_id = nullptr); + + static std::string get_temp_image_name(); + static int create_image_data_pool(std::string &data_pool); + + static std::string _local_pool_name; + static std::string _remote_pool_name; + static std::shared_ptr<librados::Rados> _rados; + static uint64_t _image_number; + static std::string _data_pool; +}; + +} // namespace mirror +} // namespace rbd + +#endif // CEPH_TEST_RBD_MIRROR_TEST_FIXTURE_H diff --git a/src/test/rbd_mirror/test_main.cc b/src/test/rbd_mirror/test_main.cc new file mode 100644 index 000000000..ed6641e2e --- /dev/null +++ b/src/test/rbd_mirror/test_main.cc @@ -0,0 +1,53 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/perf_counters.h" +#include "include/rados/librados.hpp" +#include "global/global_context.h" +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" +#include <iostream> +#include <string> + +PerfCounters *g_journal_perf_counters = nullptr; +PerfCounters *g_snapshot_perf_counters = nullptr; + +extern void register_test_cluster_watcher(); +extern void register_test_image_policy(); +extern void register_test_image_sync(); +extern void register_test_instance_watcher(); +extern void register_test_instances(); +extern void register_test_leader_watcher(); +extern void register_test_pool_watcher(); +extern void register_test_rbd_mirror(); +extern void register_test_rbd_mirror_image_deleter(); + +int main(int argc, char **argv) +{ + register_test_cluster_watcher(); + register_test_image_policy(); + register_test_image_sync(); + register_test_instance_watcher(); + register_test_instances(); + register_test_leader_watcher(); + register_test_pool_watcher(); + register_test_rbd_mirror(); + register_test_rbd_mirror_image_deleter(); + + ::testing::InitGoogleTest(&argc, argv); + + librados::Rados rados; + std::string result = connect_cluster_pp(rados); + if (result != "" ) { + std::cerr << result << std::endl; + return 1; + } + + g_ceph_context = reinterpret_cast<CephContext*>(rados.cct()); + + int r = rados.conf_set("lockdep", "true"); + if (r < 0) { + std::cerr << "warning: failed to enable lockdep" << std::endl; + } + return RUN_ALL_TESTS(); +} diff --git a/src/test/rbd_mirror/test_mock_ImageMap.cc b/src/test/rbd_mirror/test_mock_ImageMap.cc new file mode 100644 index 000000000..ac4ddb792 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_ImageMap.cc @@ -0,0 +1,1587 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "librbd/MirroringWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/ImageMap.h" +#include "tools/rbd_mirror/image_map/LoadRequest.h" +#include "tools/rbd_mirror/image_map/UpdateRequest.h" +#include "tools/rbd_mirror/image_map/Types.h" +#include "include/stringify.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +namespace image_map { + +template <> +struct LoadRequest<librbd::MockTestImageCtx> { + std::map<std::string, cls::rbd::MirrorImageMap> *image_map; + Context *on_finish = nullptr; + + static LoadRequest *s_instance; + static LoadRequest *create(librados::IoCtx &ioctx, + std::map<std::string, cls::rbd::MirrorImageMap> *image_map, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_map = image_map; + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + LoadRequest() { + s_instance = this; + } +}; + +template <> +struct UpdateRequest<librbd::MockTestImageCtx> { + Context *on_finish = nullptr; + static UpdateRequest *s_instance; + static UpdateRequest *create(librados::IoCtx &ioctx, + std::map<std::string, cls::rbd::MirrorImageMap> &&update_mapping, + std::set<std::string> &&global_image_ids, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + UpdateRequest() { + s_instance = this; + } +}; + +LoadRequest<librbd::MockTestImageCtx> * +LoadRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +UpdateRequest<librbd::MockTestImageCtx> * +UpdateRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_map + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/ImageMap.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::WithArg; +using ::testing::AtLeast; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::ReturnArg; +using ::testing::StrEq; + +using image_map::Listener; +using image_map::LoadRequest; +using image_map::UpdateRequest; + +using ::rbd::mirror::Threads; + +class TestMockImageMap : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef ImageMap<librbd::MockTestImageCtx> MockImageMap; + typedef LoadRequest<librbd::MockTestImageCtx> MockLoadRequest; + typedef UpdateRequest<librbd::MockTestImageCtx> MockUpdateRequest; + + struct MockListener : Listener { + TestMockImageMap *test_mock_image_map; + + MockListener(TestMockImageMap *test_mock_image_map) + : test_mock_image_map(test_mock_image_map) { + } + + MOCK_METHOD2(mock_acquire_image, void(const std::string &, Context*)); + MOCK_METHOD2(mock_release_image, void(const std::string &, Context*)); + MOCK_METHOD3(mock_remove_image, void(const std::string &, + const std::string &, Context*)); + + void acquire_image(const std::string &global_image_id, + const std::string &instance_id, Context* on_finish) { + mock_acquire_image(global_image_id, on_finish); + } + + void release_image(const std::string &global_image_id, + const std::string &instance_id, Context* on_finish) { + mock_release_image(global_image_id, on_finish); + } + + void remove_image(const std::string &mirror_uuid, + const std::string &global_image_id, + const std::string &instance_id, Context* on_finish) { + mock_remove_image(mirror_uuid, global_image_id, on_finish); + } + }; + + TestMockImageMap() = default; + + void SetUp() override { + TestFixture::SetUp(); + + m_local_instance_id = stringify(m_local_io_ctx.get_instance_id()); + + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", + "0")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "simple")); + } + + void TearDown() override { + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "none")); + + TestFixture::TearDown(); + } + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_,_)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = new LambdaContext([this, ctx](int r) { + std::lock_guard timer_locker{m_threads->timer_lock}; + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), ReturnArg<1>())); + } + + void expect_rebalance_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_,_)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + // disable rebalance so as to not reschedule it again + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "0"); + + auto wrapped_ctx = new LambdaContext([this, ctx](int r) { + std::lock_guard timer_locker{m_threads->timer_lock}; + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), ReturnArg<1>())); + } + + void expect_load_request(MockLoadRequest &request, int r) { + EXPECT_CALL(request, send()) + .WillOnce(Invoke([&request, r]() { + request.on_finish->complete(r); + })); + } + + void expect_update_request(MockUpdateRequest &request, int r) { + EXPECT_CALL(request, send()) + .WillOnce(Invoke([this, &request, r]() { + request.on_finish->complete(r); + if (r == 0) { + std::lock_guard locker{m_lock}; + ++m_map_update_count; + m_cond.notify_all(); + } + })); + } + + void expect_listener_acquire_image(MockListener &mock_listener, + const std::string &global_image_id, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { + std::lock_guard locker{m_lock}; + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.notify_all(); + }))); + } + + void expect_listener_release_image(MockListener &mock_listener, + const std::string &global_image_id, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_release_image(global_image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { + std::lock_guard locker{m_lock}; + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.notify_all(); + }))); + } + + void expect_listener_remove_image(MockListener &mock_listener, + const std::string &mirror_uuid, + const std::string &global_image_id, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, + mock_remove_image(mirror_uuid, global_image_id, _)) + .WillOnce(WithArg<2>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { + std::lock_guard locker{m_lock}; + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.notify_all(); + }))); + } + + void expect_listener_images_unmapped(MockListener &mock_listener, size_t count, + std::set<std::string> *global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_release_image(_, _)) + .Times(count) + .WillRepeatedly(Invoke([this, global_image_ids, peer_ack_ctxs](std::string global_image_id, Context* ctx) { + std::lock_guard locker{m_lock}; + global_image_ids->emplace(global_image_id); + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.notify_all(); + })); + } + + void remote_peer_ack_nowait(MockImageMap *image_map, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + auto ack_ctx = it->second; + peer_ack_ctxs->erase(it); + ack_ctx->complete(ret); + wait_for_scheduled_task(); + } + } + + void remote_peer_ack_wait(MockImageMap *image_map, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + auto ack_ctx = it->second; + peer_ack_ctxs->erase(it); + ack_ctx->complete(ret); + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_map_update(1)); + } + } + + void remote_peer_ack_listener_wait(MockImageMap *image_map, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + auto ack_ctx = it->second; + peer_ack_ctxs->erase(it); + ack_ctx->complete(ret); + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(1)); + } + } + + void update_map_and_acquire(MockThreads &mock_threads, + MockUpdateRequest &mock_update_request, + MockListener &mock_listener, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_add_event(mock_threads); + expect_update_request(mock_update_request, ret); + expect_add_event(mock_threads); + expect_listener_acquire_image(mock_listener, global_image_id, + peer_ack_ctxs); + } + } + + void update_map_request(MockThreads &mock_threads, + MockUpdateRequest &mock_update_request, + const std::set<std::string> &global_image_ids, int ret) { + for (uint32_t i = 0; i < global_image_ids.size(); ++i) { + expect_add_event(mock_threads); + expect_update_request(mock_update_request, ret); + } + } + + void wait_for_scheduled_task() { + m_threads->work_queue->drain(); + } + + bool wait_for_listener_notify(uint32_t count) { + std::unique_lock locker{m_lock}; + while (m_notify_update_count < count) { + if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) { + break; + } + } + + if (m_notify_update_count < count) { + return false; + } + + m_notify_update_count -= count; + return true; + } + + bool wait_for_map_update(uint32_t count) { + std::unique_lock locker{m_lock}; + while (m_map_update_count < count) { + if (m_cond.wait_for(locker, 10s) == std::cv_status::timeout) { + break; + } + } + + if (m_map_update_count < count) { + return false; + } + + m_map_update_count -= count; + return true; + } + + int when_shut_down(MockImageMap *image_map) { + C_SaferCond ctx; + image_map->shut_down(&ctx); + return ctx.wait(); + } + + void listener_acquire_images(MockListener &mock_listener, + const std::set<std::string> &global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_listener_acquire_image(mock_listener, global_image_id, + peer_ack_ctxs); + } + } + + void listener_release_images(MockListener &mock_listener, + const std::set<std::string> &global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_listener_release_image(mock_listener, global_image_id, + peer_ack_ctxs); + } + } + + void listener_remove_images(MockListener &mock_listener, + const std::string &mirror_uuid, + std::set<std::string> &global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id, + peer_ack_ctxs); + } + } + + ceph::mutex m_lock = ceph::make_mutex("TestMockImageMap::m_lock"); + ceph::condition_variable m_cond; + uint32_t m_notify_update_count = 0; + uint32_t m_map_update_count = 0; + std::string m_local_instance_id; +}; + +TEST_F(TestMockImageMap, SetLocalImages) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddRemoveLocalImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("", std::move(initial_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // RELEASE+REMOVE_MAPPING + expect_add_event(mock_threads); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, + 0); + + // remove images + mock_image_map->update_images("", {}, std::move(remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size())); + + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddRemoveRemoteImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), + {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // RELEASE+REMOVE_MAPPING + std::map<std::string, Context*> peer_remove_ack_ctxs; + listener_remove_images(mock_listener, "uuid1", remove_global_image_ids, + &peer_remove_ack_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, + 0); + + // remove images + mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_remove_ack_ctxs); + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_dup(initial_global_image_ids); + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> remove_global_image_ids_dup(remove_global_image_ids); + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // trigger duplicate "add" event + wait_for_scheduled_task(); + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids_dup), {}); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // RELEASE+REMOVE_MAPPING + std::map<std::string, Context*> peer_remove_ack_ctxs; + listener_remove_images(mock_listener, "uuid1", remove_global_image_ids, + &peer_remove_ack_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0); + + // remove images + mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_remove_ack_ctxs); + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // trigger duplicate "remove" notification + mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids_dup)); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AcquireImageErrorRetry) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + // UPDATE_MAPPING failure + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, -EIO); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + // remote image set + std::set<std::string> initial_remote_global_image_ids{ + "global id 1" + }; + std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids); + + // local image set + std::set<std::string> initial_local_global_image_ids{ + "global id 1" + }; + + // remote/local images to remove + std::set<std::string> remote_remove_global_image_ids{ + "global id 1" + }; + std::set<std::string> remote_remove_global_image_ids_ack(remote_remove_global_image_ids); + + std::set<std::string> local_remove_global_image_ids{ + "global id 1" + }; + std::set<std::string> local_remove_global_image_ids_ack(local_remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_remote_global_image_ids, + &peer_ack_ctxs); + + // initial remote image list + mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), + initial_remote_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // set initial local image list -- this is a no-op from policy pov + mock_image_map->update_images("", std::move(initial_local_global_image_ids), {}); + + // remove remote images -- this should be a no-op from policy pov + // except the listener notification + std::map<std::string, Context*> peer_ack_remove_ctxs; + listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids, + &peer_ack_remove_ctxs); + + mock_image_map->update_images("uuid1", {}, std::move(remote_remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remote_remove_global_image_ids_ack.size())); + + // RELEASE+REMOVE_MAPPING + expect_add_event(mock_threads); + listener_release_images(mock_listener, local_remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, local_remove_global_image_ids, 0); + + // remove local images + mock_image_map->update_images("", {}, std::move(local_remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(local_remove_global_image_ids_ack.size())); + + remote_peer_ack_nowait(mock_image_map.get(), local_remove_global_image_ids_ack, + 0, &peer_ack_remove_ctxs); + remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack, + 0, &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddInstance) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5" + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RemoveInstance) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5" + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + expect_add_event(mock_threads); + + // UPDATE_MAPPING+ACQUIRE + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // set initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request -- completing action + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + shuffled_global_image_ids.clear(); + + // remove added instance + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_removed({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddInstancePingPongImageTest) { + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", "600")); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6", "global id 7", "global id 8", "global id 9", "global id 10", + "global id 11", "global id 12", "global id 13", "global id 14" + }; + + std::map<std::string, cls::rbd::MirrorImageMap> image_mapping; + for (auto& global_image_id : global_image_ids) { + image_mapping[global_image_id] = {m_local_instance_id, {}, {}}; + } + + // ACQUIRE + MockLoadRequest mock_load_request; + EXPECT_CALL(mock_load_request, send()).WillOnce( + Invoke([&mock_load_request, &image_mapping]() { + *mock_load_request.image_map = image_mapping; + mock_load_request.on_finish->complete(0); + })); + + expect_add_event(mock_threads); + MockListener mock_listener(this); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> global_image_ids_ack(global_image_ids); + + // remote peer ACKs image acquire request -- completing action + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // set initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request -- completing action + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 7, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> migrated_global_image_ids(shuffled_global_image_ids); + shuffled_global_image_ids.clear(); + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids, + &peer_ack_ctxs); + + // add another instance + mock_image_map->update_instances_added({"5432"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + // shuffle set should be distinct + std::set<std::string> reshuffled; + std::set_intersection(migrated_global_image_ids.begin(), migrated_global_image_ids.end(), + shuffled_global_image_ids.begin(), shuffled_global_image_ids.end(), + std::inserter(reshuffled, reshuffled.begin())); + ASSERT_TRUE(reshuffled.empty()); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "remote id 4", + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1" + }; + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + expect_add_event(mock_threads); + // UPDATE_MAPPING+ACQUIRE + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids); + + // RELEASE + + std::map<std::string, Context*> peer_ack_remove_ctxs; + listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids, + &peer_ack_remove_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + + mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2)); + + // instance failed -- update policy for instance removal + mock_image_map->update_instances_removed({"9876"}); + + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, + -ENOENT, &peer_ack_remove_ctxs); + remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids, + -EBLOCKLISTED, &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddErrorAndRemoveImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "remote id 4", + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_removed({"9876"}); + + std::set<std::string> released_global_image_ids; + std::map<std::string, Context*> release_peer_ack_ctxs; + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids, + &release_peer_ack_ctxs); + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids, + &release_peer_ack_ctxs); + + // instance blocklisted -- ACQUIRE request fails + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, + -EBLOCKLISTED, &peer_ack_ctxs); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + std::map<std::string, Context*> remap_peer_ack_ctxs; + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &remap_peer_ack_ctxs); + + // instance blocklisted -- RELEASE request fails + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + -ENOENT, &release_peer_ack_ctxs); + wait_for_scheduled_task(); + + // new peer acks acquire request + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &remap_peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids); + + // remove image + std::map<std::string, Context*> peer_ack_remove_ctxs; + listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids, + &peer_ack_remove_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, 0); + + mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids_ack, 0, + &peer_ack_remove_ctxs); + remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, MirrorUUIDUpdated) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + // remote image set + std::set<std::string> initial_remote_global_image_ids{ + "global id 1", "global id 2", "global id 3" + }; + std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids); + + // remote/local images to remove + std::set<std::string> remote_removed_global_image_ids{ + "global id 1", "global id 2", "global id 3" + }; + std::set<std::string> remote_removed_global_image_ids_ack(remote_removed_global_image_ids); + + std::set<std::string> remote_added_global_image_ids{ + "global id 1", "global id 2", "global id 3" + }; + std::set<std::string> remote_added_global_image_ids_ack(remote_added_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_remote_global_image_ids, + &peer_ack_ctxs); + + // initial remote image list + mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), + initial_remote_global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + // RELEASE+REMOVE_MAPPING + std::map<std::string, Context*> peer_remove_ack_ctxs; + listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids, + &peer_remove_ack_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, remote_removed_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remote_removed_global_image_ids, 0); + + mock_image_map->update_images("uuid1", {}, std::move(remote_removed_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remote_removed_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), + remote_removed_global_image_ids_ack, 0, + &peer_remove_ack_ctxs); + remote_peer_ack_wait(mock_image_map.get(), + remote_removed_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + listener_acquire_images(mock_listener, remote_added_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_images("uuid2", std::move(remote_added_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(remote_added_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), + remote_added_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RebalanceImageMap) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6", "global id 7", "global id 8", "global id 9", "global id 10", + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 5, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + + // remove all shuffled images -- make way for rebalance + std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids); + + // RELEASE+REMOVE_MAPPING + expect_add_event(mock_threads); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, + 0); + + mock_image_map->update_images("", {}, std::move(shuffled_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size())); + + remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + shuffled_global_image_ids.clear(); + shuffled_global_image_ids_ack.clear(); + + std::set<std::string> new_global_image_ids = { + "global id 11" + }; + std::set<std::string> new_global_image_ids_ack(new_global_image_ids); + + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + listener_acquire_images(mock_listener, new_global_image_ids, &peer_ack_ctxs); + + expect_rebalance_event(mock_threads); // rebalance task + expect_add_event(mock_threads); // update task scheduled by + // rebalance task + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_images("", std::move(new_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(new_global_image_ids_ack.size())); + + // set rebalance interval + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "5"); + remote_peer_ack_nowait(mock_image_map.get(), new_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc new file mode 100644 index 000000000..177b71a15 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -0,0 +1,950 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/MirrorStatusUpdater.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/BootstrapRequest.h" +#include "tools/rbd_mirror/image_replayer/Replayer.h" +#include "tools/rbd_mirror/image_replayer/ReplayerListener.h" +#include "tools/rbd_mirror/image_replayer/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct ImageDeleter<librbd::MockTestImageCtx> { + static ImageDeleter* s_instance; + + static void trash_move(librados::IoCtx& local_io_ctx, + const std::string& global_image_id, bool resync, + MockContextWQ* work_queue, Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->trash_move(global_image_id, resync, on_finish); + } + + MOCK_METHOD3(trash_move, void(const std::string&, bool, Context*)); + + ImageDeleter() { + s_instance = this; + } +}; + +ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr; + +template <> +struct MirrorStatusUpdater<librbd::MockTestImageCtx> { + + MOCK_METHOD1(exists, bool(const std::string&)); + MOCK_METHOD3(set_mirror_image_status, + void(const std::string&, const cls::rbd::MirrorImageSiteStatus&, + bool)); + MOCK_METHOD2(remove_refresh_mirror_image_status, void(const std::string&, + Context*)); + MOCK_METHOD3(remove_mirror_image_status, void(const std::string&, bool, + Context*)); +}; + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +template<> +class InstanceWatcher<librbd::MockTestImageCtx> { +}; + +namespace image_replayer { + +template<> +struct BootstrapRequest<librbd::MockTestImageCtx> { + static BootstrapRequest* s_instance; + + StateBuilder<librbd::MockTestImageCtx>** state_builder = nullptr; + bool *do_resync = nullptr; + Context *on_finish = nullptr; + + static BootstrapRequest* create( + Threads<librbd::MockTestImageCtx>* threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx& remote_io_ctx, + rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const RemotePoolMeta& remote_pool_meta, + ::journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache, + rbd::mirror::ProgressContext *progress_ctx, + StateBuilder<librbd::MockTestImageCtx>** state_builder, + bool *do_resync, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->state_builder = state_builder; + s_instance->do_resync = do_resync; + s_instance->on_finish = on_finish; + return s_instance; + } + + BootstrapRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~BootstrapRequest() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + void put() { + } + + void get() { + } + + std::string get_local_image_name() const { + return "local image name"; + } + + inline bool is_syncing() const { + return false; + } + + MOCK_METHOD0(send, void()); + MOCK_METHOD0(cancel, void()); +}; + +struct MockReplayer : public Replayer { + image_replayer::ReplayerListener* replayer_listener; + + MOCK_METHOD0(destroy, void()); + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + MOCK_METHOD1(flush, void(Context*)); + + MOCK_METHOD2(get_replay_status, bool(std::string*, Context*)); + + MOCK_CONST_METHOD0(is_replaying, bool()); + MOCK_CONST_METHOD0(is_resync_requested, bool()); + MOCK_CONST_METHOD0(get_error_code, int()); + MOCK_CONST_METHOD0(get_error_description, std::string()); +}; + +template <> +struct StateBuilder<librbd::MockTestImageCtx> { + static StateBuilder* s_instance; + + librbd::MockTestImageCtx* local_image_ctx = nullptr; + std::string local_image_id; + std::string remote_image_id; + + void destroy() { + } + + MOCK_METHOD1(close, void(Context*)); + MOCK_METHOD5(create_replayer, Replayer*(Threads<librbd::MockTestImageCtx>*, + InstanceWatcher<librbd::MockTestImageCtx>*, + const std::string&, PoolMetaCache*, + ReplayerListener*)); + + StateBuilder() { + s_instance = this; + } +}; + +BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +StateBuilder<librbd::MockTestImageCtx>* StateBuilder<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/ImageReplayer.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::MatcherCast; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::SetArgPointee; +using ::testing::WithArg; + +class TestMockImageReplayer : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter; + typedef MirrorStatusUpdater<librbd::MockTestImageCtx> MockMirrorStatusUpdater; + typedef image_replayer::BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest; + typedef image_replayer::StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef image_replayer::MockReplayer MockReplayer; + typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void TearDown() override { + delete m_image_replayer; + + TestMockFixture::TearDown(); + } + + void create_local_image() { + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_work_queue_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event_after_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillRepeatedly( + DoAll(Invoke([this](double seconds, Context *ctx) { + m_threads->timer->add_event_after(seconds, ctx); + }), + ReturnArg<1>())); + EXPECT_CALL(*mock_threads.timer, cancel_event(_)) + .WillRepeatedly( + Invoke([this](Context *ctx) { + return m_threads->timer->cancel_event(ctx); + })); + } + + void expect_trash_move(MockImageDeleter& mock_image_deleter, + const std::string& global_image_id, + bool ignore_orphan, int r) { + EXPECT_CALL(mock_image_deleter, + trash_move(global_image_id, ignore_orphan, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { + bufferlist bl; + encode(tag_data, bl); + return bl; + } + + void expect_send(MockBootstrapRequest& mock_bootstrap_request, + MockStateBuilder& mock_state_builder, + librbd::MockTestImageCtx& mock_local_image_ctx, + bool do_resync, bool set_local_image, int r) { + EXPECT_CALL(mock_bootstrap_request, send()) + .WillOnce(Invoke([this, &mock_bootstrap_request, &mock_state_builder, + &mock_local_image_ctx, set_local_image, do_resync, + r]() { + if (r == 0 || r == -ENOLINK) { + mock_state_builder.local_image_id = mock_local_image_ctx.id; + mock_state_builder.remote_image_id = m_remote_image_ctx->id; + *mock_bootstrap_request.state_builder = &mock_state_builder; + } + if (r == 0) { + mock_state_builder.local_image_ctx = &mock_local_image_ctx; + *mock_bootstrap_request.do_resync = do_resync; + } + if (r < 0 && r != -ENOENT) { + mock_state_builder.remote_image_id = ""; + } + if (r == -ENOENT) { + *mock_bootstrap_request.state_builder = &mock_state_builder; + } + if (set_local_image) { + mock_state_builder.local_image_id = mock_local_image_ctx.id; + } + mock_bootstrap_request.on_finish->complete(r); + })); + } + + void expect_create_replayer(MockStateBuilder& mock_state_builder, + MockReplayer& mock_replayer) { + EXPECT_CALL(mock_state_builder, create_replayer(_, _, _, _, _)) + .WillOnce(WithArg<4>( + Invoke([&mock_replayer] + (image_replayer::ReplayerListener* replayer_listener) { + mock_replayer.replayer_listener = replayer_listener; + return &mock_replayer; + }))); + } + + void expect_close(MockStateBuilder& mock_state_builder, int r) { + EXPECT_CALL(mock_state_builder, close(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_init(MockReplayer& mock_replayer, int r) { + EXPECT_CALL(mock_replayer, init(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_shut_down(MockReplayer& mock_replayer, int r) { + EXPECT_CALL(mock_replayer, shut_down(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + EXPECT_CALL(mock_replayer, destroy()); + } + + void expect_get_replay_status(MockReplayer& mock_replayer) { + EXPECT_CALL(mock_replayer, get_replay_status(_, _)) + .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)), + Return(true))); + } + + void expect_set_mirror_image_status_repeatedly() { + EXPECT_CALL(m_local_status_updater, set_mirror_image_status(_, _, _)) + .WillRepeatedly(Invoke([](auto, auto, auto){})); + EXPECT_CALL(m_remote_status_updater, set_mirror_image_status(_, _, _)) + .WillRepeatedly(Invoke([](auto, auto, auto){})); + } + + void expect_mirror_image_status_exists(bool exists) { + EXPECT_CALL(m_local_status_updater, exists(_)) + .WillOnce(Return(exists)); + EXPECT_CALL(m_remote_status_updater, exists(_)) + .WillOnce(Return(exists)); + } + + void create_image_replayer(MockThreads &mock_threads) { + m_image_replayer = new MockImageReplayer( + m_local_io_ctx, "local_mirror_uuid", "global image id", + &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr, + nullptr); + m_image_replayer->add_peer({"peer_uuid", m_remote_io_ctx, + {"remote mirror uuid", + "remote mirror peer uuid"}, + &m_remote_status_updater}); + } + + void wait_for_stopped() { + for (int i = 0; i < 10000; i++) { + if (m_image_replayer->is_stopped()) { + break; + } + usleep(1000); + } + ASSERT_TRUE(m_image_replayer->is_stopped()); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx = nullptr; + MockInstanceWatcher m_instance_watcher; + MockMirrorStatusUpdater m_local_status_updater; + MockMirrorStatusUpdater m_remote_status_updater; + MockImageReplayer *m_image_replayer = nullptr; +}; + +TEST_F(TestMockImageReplayer, StartStop) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockReplayer mock_replayer; + + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockBootstrapRequest mock_bootstrap_request; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + ASSERT_EQ(image_replayer::HEALTH_STATE_OK, + m_image_replayer->get_health_state()); + + // STOP + expect_shut_down(mock_replayer, 0); + expect_close(mock_state_builder, 0); + expect_mirror_image_status_exists(false); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(0, stop_ctx.wait()); + ASSERT_EQ(image_replayer::HEALTH_STATE_OK, + m_image_replayer->get_health_state()); +} + +TEST_F(TestMockImageReplayer, LocalImagePrimary) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, -ENOMSG); + + expect_mirror_image_status_exists(false); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, MetadataCleanup) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + MockReplayer mock_replayer; + + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, true, -ENOLINK); + + expect_close(mock_state_builder, 0); + expect_trash_move(mock_image_deleter, "global image id", false, 0); + expect_mirror_image_status_exists(false); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, BootstrapRemoteDeleted) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + + MockBootstrapRequest mock_bootstrap_request; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, -ENOLINK); + + expect_close(mock_state_builder, 0); + + expect_trash_move(mock_image_deleter, "global image id", false, 0); + expect_mirror_image_status_exists(false); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, BootstrapResyncRequested) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + + MockBootstrapRequest mock_bootstrap_request; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + true, false, 0); + + expect_close(mock_state_builder, 0); + + expect_trash_move(mock_image_deleter, "global image id", true, 0); + expect_mirror_image_status_exists(false); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, BootstrapError) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, -EINVAL); + + expect_mirror_image_status_exists(false); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EINVAL, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, BootstrapCancel) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + + create_image_replayer(mock_threads); + + MockBootstrapRequest mock_bootstrap_request; + MockStateBuilder mock_state_builder; + EXPECT_CALL(mock_bootstrap_request, send()) + .WillOnce(Invoke([this, &mock_bootstrap_request]() { + m_image_replayer->stop(nullptr); + mock_bootstrap_request.on_finish->complete(-ECANCELED); + })); + EXPECT_CALL(mock_bootstrap_request, cancel()); + + expect_mirror_image_status_exists(false); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-ECANCELED, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, StopError) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + MockReplayer mock_replayer; + + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // STOP (errors are ignored) + + expect_shut_down(mock_replayer, -EINVAL); + expect_close(mock_state_builder, -EINVAL); + expect_mirror_image_status_exists(false); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(0, stop_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, ReplayerError) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + MockReplayer mock_replayer; + + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, -EINVAL); + EXPECT_CALL(mock_replayer, get_error_description()) + .WillOnce(Return("FAIL")); + + EXPECT_CALL(mock_replayer, destroy()); + expect_close(mock_state_builder, -EINVAL); + + expect_mirror_image_status_exists(false); + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EINVAL, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, ReplayerResync) { + // START + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + MockReplayer mock_replayer; + + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // NOTIFY + EXPECT_CALL(mock_replayer, is_resync_requested()) + .WillOnce(Return(true)); + expect_shut_down(mock_replayer, 0); + expect_close(mock_state_builder, 0); + expect_trash_move(mock_image_deleter, "global image id", true, 0); + expect_mirror_image_status_exists(false); + mock_replayer.replayer_listener->handle_notification(); + ASSERT_FALSE(m_image_replayer->is_running()); + + wait_for_stopped(); +} + +TEST_F(TestMockImageReplayer, ReplayerInterrupted) { + // START + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + MockReplayer mock_replayer; + + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // NOTIFY + EXPECT_CALL(mock_replayer, is_resync_requested()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_replayer, is_replaying()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_replayer, get_error_code()) + .WillOnce(Return(-EINVAL)); + EXPECT_CALL(mock_replayer, get_error_description()) + .WillOnce(Return("INVALID")); + expect_shut_down(mock_replayer, 0); + expect_close(mock_state_builder, 0); + expect_mirror_image_status_exists(false); + mock_replayer.replayer_listener->handle_notification(); + ASSERT_FALSE(m_image_replayer->is_running()); + + wait_for_stopped(); +} + +TEST_F(TestMockImageReplayer, ReplayerRenamed) { + // START + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + MockReplayer mock_replayer; + + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // NOTIFY + EXPECT_CALL(mock_replayer, is_resync_requested()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_replayer, is_replaying()) + .WillOnce(Return(true)); + mock_local_image_ctx.name = "NEW NAME"; + mock_replayer.replayer_listener->handle_notification(); + + // STOP + expect_shut_down(mock_replayer, 0); + expect_close(mock_state_builder, 0); + expect_mirror_image_status_exists(false); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(0, stop_ctx.wait()); + + auto image_spec = image_replayer::util::compute_image_spec( + m_local_io_ctx, "NEW NAME"); + ASSERT_EQ(image_spec, m_image_replayer->get_name()); +} + +TEST_F(TestMockImageReplayer, StopJoinInterruptedReplayer) { + // START + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockReplayer mock_replayer; + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockBootstrapRequest mock_bootstrap_request; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // NOTIFY + EXPECT_CALL(mock_replayer, is_resync_requested()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_replayer, is_replaying()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_replayer, get_error_code()) + .WillOnce(Return(-EINVAL)); + EXPECT_CALL(mock_replayer, get_error_description()) + .WillOnce(Return("INVALID")); + const double DELAY = 10; + EXPECT_CALL(mock_replayer, shut_down(_)) + .WillOnce(Invoke([this, DELAY](Context* ctx) { + std::lock_guard l(m_threads->timer_lock); + m_threads->timer->add_event_after(DELAY, ctx); + })); + EXPECT_CALL(mock_replayer, destroy()); + expect_close(mock_state_builder, 0); + expect_mirror_image_status_exists(false); + + mock_replayer.replayer_listener->handle_notification(); + ASSERT_FALSE(m_image_replayer->is_running()); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(ETIMEDOUT, stop_ctx.wait_for(DELAY * 3 / 4)); + ASSERT_EQ(0, stop_ctx.wait_for(DELAY)); +} + +TEST_F(TestMockImageReplayer, StopJoinRequestedStop) { + // START + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockReplayer mock_replayer; + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + MockBootstrapRequest mock_bootstrap_request; + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, false, 0); + + expect_create_replayer(mock_state_builder, mock_replayer); + expect_init(mock_replayer, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // STOP + const double DELAY = 10; + EXPECT_CALL(mock_replayer, shut_down(_)) + .WillOnce(Invoke([this, DELAY](Context* ctx) { + std::lock_guard l(m_threads->timer_lock); + m_threads->timer->add_event_after(DELAY, ctx); + })); + EXPECT_CALL(mock_replayer, destroy()); + expect_close(mock_state_builder, 0); + expect_mirror_image_status_exists(false); + + C_SaferCond stop_ctx1; + m_image_replayer->stop(&stop_ctx1); + + C_SaferCond stop_ctx2; + m_image_replayer->stop(&stop_ctx2); + ASSERT_EQ(ETIMEDOUT, stop_ctx2.wait_for(DELAY * 3 / 4)); + ASSERT_EQ(0, stop_ctx2.wait_for(DELAY)); + + ASSERT_EQ(0, stop_ctx1.wait_for(0)); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_ImageSync.cc b/src/test/rbd_mirror/test_mock_ImageSync.cc new file mode 100644 index 000000000..bd6a29078 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_ImageSync.cc @@ -0,0 +1,468 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "librbd/DeepCopyRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/image_sync/MockSyncPointHandler.h" +#include "tools/rbd_mirror/ImageSync.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" +#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +template <> +class DeepCopyRequest<librbd::MockTestImageCtx> { +public: + static DeepCopyRequest* s_instance; + Context *on_finish; + + static DeepCopyRequest* create( + librbd::MockTestImageCtx *src_image_ctx, + librbd::MockTestImageCtx *dst_image_ctx, + librados::snap_t src_snap_id_start, librados::snap_t src_snap_id_end, + librados::snap_t dst_snap_id_start, bool flatten, + const librbd::deep_copy::ObjectNumber &object_number, + librbd::asio::ContextWQ *work_queue, SnapSeqs *snap_seqs, + deep_copy::Handler *handler, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + DeepCopyRequest() { + s_instance = this; + } + + void put() { + } + + void get() { + } + + MOCK_METHOD0(cancel, void()); + MOCK_METHOD0(send, void()); +}; + +DeepCopyRequest<librbd::MockTestImageCtx>* DeepCopyRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/ImageSync.cc" + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { + MOCK_METHOD2(notify_sync_request, void(const std::string, Context *)); + MOCK_METHOD1(cancel_sync_request, bool(const std::string &)); + MOCK_METHOD1(notify_sync_complete, void(const std::string &)); +}; + +namespace image_sync { + +template <> +class SyncPointCreateRequest<librbd::MockTestImageCtx> { +public: + static SyncPointCreateRequest *s_instance; + Context *on_finish; + + static SyncPointCreateRequest* create(librbd::MockTestImageCtx *remote_image_ctx, + const std::string &mirror_uuid, + image_sync::SyncPointHandler* sync_point_handler, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + SyncPointCreateRequest() { + s_instance = this; + } + MOCK_METHOD0(send, void()); +}; + +template <> +class SyncPointPruneRequest<librbd::MockTestImageCtx> { +public: + static SyncPointPruneRequest *s_instance; + Context *on_finish; + bool sync_complete; + + static SyncPointPruneRequest* create(librbd::MockTestImageCtx *remote_image_ctx, + bool sync_complete, + image_sync::SyncPointHandler* sync_point_handler, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + s_instance->sync_complete = sync_complete; + return s_instance; + } + + SyncPointPruneRequest() { + s_instance = this; + } + MOCK_METHOD0(send, void()); +}; + +SyncPointCreateRequest<librbd::MockTestImageCtx>* SyncPointCreateRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +SyncPointPruneRequest<librbd::MockTestImageCtx>* SyncPointPruneRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_sync + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::InvokeWithoutArgs; + +class TestMockImageSync : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef ImageSync<librbd::MockTestImageCtx> MockImageSync; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef image_sync::SyncPointCreateRequest<librbd::MockTestImageCtx> MockSyncPointCreateRequest; + typedef image_sync::SyncPointPruneRequest<librbd::MockTestImageCtx> MockSyncPointPruneRequest; + typedef image_sync::MockSyncPointHandler MockSyncPointHandler; + typedef librbd::DeepCopyRequest<librbd::MockTestImageCtx> MockImageCopyRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_get_snap_id(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, get_snap_id(_, _)) + .WillOnce(Return(123)); + } + + void expect_notify_sync_request(MockInstanceWatcher &mock_instance_watcher, + const std::string &sync_id, int r) { + EXPECT_CALL(mock_instance_watcher, notify_sync_request(sync_id, _)) + .WillOnce(Invoke([this, r](const std::string &, Context *on_sync_start) { + m_threads->work_queue->queue(on_sync_start, r); + })); + } + + void expect_cancel_sync_request(MockInstanceWatcher &mock_instance_watcher, + const std::string &sync_id, bool canceled) { + EXPECT_CALL(mock_instance_watcher, cancel_sync_request(sync_id)) + .WillOnce(Return(canceled)); + } + + void expect_notify_sync_complete(MockInstanceWatcher &mock_instance_watcher, + const std::string &sync_id) { + EXPECT_CALL(mock_instance_watcher, notify_sync_complete(sync_id)); + } + + void expect_create_sync_point(librbd::MockTestImageCtx &mock_local_image_ctx, + MockSyncPointCreateRequest &mock_sync_point_create_request, + int r) { + EXPECT_CALL(mock_sync_point_create_request, send()) + .WillOnce(Invoke([this, &mock_local_image_ctx, &mock_sync_point_create_request, r]() { + if (r == 0) { + mock_local_image_ctx.snap_ids[{cls::rbd::UserSnapshotNamespace(), + "snap1"}] = 123; + m_sync_points.emplace_back(cls::rbd::UserSnapshotNamespace(), + "snap1", "", boost::none); + } + m_threads->work_queue->queue(mock_sync_point_create_request.on_finish, r); + })); + } + + void expect_copy_image(MockImageCopyRequest &mock_image_copy_request, int r) { + EXPECT_CALL(mock_image_copy_request, send()) + .WillOnce(Invoke([this, &mock_image_copy_request, r]() { + m_threads->work_queue->queue(mock_image_copy_request.on_finish, r); + })); + } + + void expect_flush_sync_point(MockSyncPointHandler& mock_sync_point_handler, + int r) { + EXPECT_CALL(mock_sync_point_handler, update_sync_points(_, _, false, _)) + .WillOnce(WithArg<3>(CompleteContext(r))); + } + + void expect_prune_sync_point(MockSyncPointPruneRequest &mock_sync_point_prune_request, + bool sync_complete, int r) { + EXPECT_CALL(mock_sync_point_prune_request, send()) + .WillOnce(Invoke([this, &mock_sync_point_prune_request, sync_complete, r]() { + ASSERT_EQ(sync_complete, mock_sync_point_prune_request.sync_complete); + if (r == 0 && !m_sync_points.empty()) { + if (sync_complete) { + m_sync_points.pop_front(); + } else { + while (m_sync_points.size() > 1) { + m_sync_points.pop_back(); + } + } + } + m_threads->work_queue->queue(mock_sync_point_prune_request.on_finish, r); + })); + } + + void expect_get_snap_seqs(MockSyncPointHandler& mock_sync_point_handler) { + EXPECT_CALL(mock_sync_point_handler, get_snap_seqs()) + .WillRepeatedly(Return(librbd::SnapSeqs{})); + } + + void expect_get_sync_points(MockSyncPointHandler& mock_sync_point_handler) { + EXPECT_CALL(mock_sync_point_handler, get_sync_points()) + .WillRepeatedly(Invoke([this]() { + return m_sync_points; + })); + } + + MockImageSync *create_request(MockThreads& mock_threads, + librbd::MockTestImageCtx &mock_remote_image_ctx, + librbd::MockTestImageCtx &mock_local_image_ctx, + MockSyncPointHandler& mock_sync_point_handler, + MockInstanceWatcher &mock_instance_watcher, + Context *ctx) { + return new MockImageSync(&mock_threads, &mock_local_image_ctx, + &mock_remote_image_ctx, + "mirror-uuid", &mock_sync_point_handler, + &mock_instance_watcher, nullptr, ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx; + + image_sync::SyncPoints m_sync_points; +}; + +TEST_F(TestMockImageSync, SimpleSync) { + MockThreads mock_threads(m_threads); + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_create_sync_point(mock_local_image_ctx, mock_sync_point_create_request, 0); + expect_get_snap_id(mock_remote_image_ctx); + expect_copy_image(mock_image_copy_request, 0); + expect_flush_sync_point(mock_sync_point_handler, 0); + expect_prune_sync_point(mock_sync_point_prune_request, true, 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_threads, mock_remote_image_ctx, + mock_local_image_ctx, + mock_sync_point_handler, + mock_instance_watcher, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageSync, RestartSync) { + MockThreads mock_threads(m_threads); + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + m_sync_points = {{cls::rbd::UserSnapshotNamespace(), "snap1", "", boost::none}, + {cls::rbd::UserSnapshotNamespace(), "snap2", "snap1", boost::none}}; + mock_local_image_ctx.snap_ids[{cls::rbd::UserSnapshotNamespace(), "snap1"}] = 123; + mock_local_image_ctx.snap_ids[{cls::rbd::UserSnapshotNamespace(), "snap2"}] = 234; + + expect_test_features(mock_local_image_ctx); + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_prune_sync_point(mock_sync_point_prune_request, false, 0); + expect_get_snap_id(mock_remote_image_ctx); + expect_copy_image(mock_image_copy_request, 0); + expect_flush_sync_point(mock_sync_point_handler, 0); + expect_prune_sync_point(mock_sync_point_prune_request, true, 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_threads, mock_remote_image_ctx, + mock_local_image_ctx, + mock_sync_point_handler, + mock_instance_watcher, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageSync, CancelNotifySyncRequest) { + MockThreads mock_threads(m_threads); + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + MockInstanceWatcher mock_instance_watcher; + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + Context *on_sync_start = nullptr; + C_SaferCond notify_sync_ctx; + EXPECT_CALL(mock_instance_watcher, + notify_sync_request(mock_local_image_ctx.id, _)) + .WillOnce(Invoke([&on_sync_start, ¬ify_sync_ctx]( + const std::string &, Context *ctx) { + on_sync_start = ctx; + notify_sync_ctx.complete(0); + })); + EXPECT_CALL(mock_instance_watcher, + cancel_sync_request(mock_local_image_ctx.id)) + .WillOnce(Invoke([&on_sync_start](const std::string &) { + EXPECT_NE(nullptr, on_sync_start); + on_sync_start->complete(-ECANCELED); + return true; + })); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_threads, mock_remote_image_ctx, + mock_local_image_ctx, + mock_sync_point_handler, + mock_instance_watcher, &ctx); + request->get(); + request->send(); + + // cancel the notify sync request once it starts + ASSERT_EQ(0, notify_sync_ctx.wait()); + request->cancel(); + request->put(); + + ASSERT_EQ(-ECANCELED, ctx.wait()); +} + +TEST_F(TestMockImageSync, CancelImageCopy) { + MockThreads mock_threads(m_threads); + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + m_sync_points = {{cls::rbd::UserSnapshotNamespace(), "snap1", "", boost::none}}; + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_prune_sync_point(mock_sync_point_prune_request, false, 0); + expect_get_snap_id(mock_remote_image_ctx); + + C_SaferCond image_copy_ctx; + EXPECT_CALL(mock_image_copy_request, send()) + .WillOnce(Invoke([&image_copy_ctx]() { + image_copy_ctx.complete(0); + })); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id, + false); + EXPECT_CALL(mock_image_copy_request, cancel()); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_threads, mock_remote_image_ctx, + mock_local_image_ctx, + mock_sync_point_handler, + mock_instance_watcher, &ctx); + request->get(); + request->send(); + + // cancel the image copy once it starts + ASSERT_EQ(0, image_copy_ctx.wait()); + request->cancel(); + request->put(); + m_threads->work_queue->queue(mock_image_copy_request.on_finish, 0); + + ASSERT_EQ(-ECANCELED, ctx.wait()); +} + +TEST_F(TestMockImageSync, CancelAfterCopyImage) { + MockThreads mock_threads(m_threads); + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + MockSyncPointHandler mock_sync_point_handler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_threads, mock_remote_image_ctx, + mock_local_image_ctx, + mock_sync_point_handler, + mock_instance_watcher, &ctx); + + expect_get_snap_seqs(mock_sync_point_handler); + expect_get_sync_points(mock_sync_point_handler); + + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_create_sync_point(mock_local_image_ctx, mock_sync_point_create_request, 0); + expect_get_snap_id(mock_remote_image_ctx); + EXPECT_CALL(mock_image_copy_request, send()) + .WillOnce((DoAll(InvokeWithoutArgs([request]() { + request->cancel(); + }), + Invoke([this, &mock_image_copy_request]() { + m_threads->work_queue->queue(mock_image_copy_request.on_finish, 0); + })))); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id, + false); + EXPECT_CALL(mock_image_copy_request, cancel()); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + request->send(); + ASSERT_EQ(-ECANCELED, ctx.wait()); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc new file mode 100644 index 000000000..1cc64be60 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -0,0 +1,382 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/InstanceReplayer.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/Types.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + ceph::condition_variable timer_cond; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +template<> +struct ServiceDaemon<librbd::MockTestImageCtx> { + MOCK_METHOD4(add_or_update_namespace_attribute, + void(int64_t, const std::string&, const std::string&, + const service_daemon::AttributeValue&)); +}; + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { +}; + +template<> +struct ImageReplayer<librbd::MockTestImageCtx> { + static ImageReplayer* s_instance; + std::string global_image_id; + + static ImageReplayer *create( + librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid, + const std::string &global_image_id, + Threads<librbd::MockTestImageCtx> *threads, + InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher, + MirrorStatusUpdater<librbd::MockTestImageCtx>* local_status_updater, + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { + ceph_assert(s_instance != nullptr); + s_instance->global_image_id = global_image_id; + return s_instance; + } + + ImageReplayer() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + virtual ~ImageReplayer() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD2(start, void(Context *, bool)); + MOCK_METHOD2(stop, void(Context *, bool)); + MOCK_METHOD1(restart, void(Context*)); + MOCK_METHOD0(flush, void()); + MOCK_METHOD1(print_status, void(Formatter *)); + MOCK_METHOD1(add_peer, void(const Peer<librbd::MockTestImageCtx>& peer)); + MOCK_METHOD0(get_global_image_id, const std::string &()); + MOCK_METHOD0(get_local_image_id, const std::string &()); + MOCK_METHOD0(is_running, bool()); + MOCK_METHOD0(is_stopped, bool()); + MOCK_METHOD0(is_blocklisted, bool()); + + MOCK_CONST_METHOD0(is_finished, bool()); + MOCK_METHOD1(set_finished, void(bool)); + + MOCK_CONST_METHOD0(get_health_state, image_replayer::HealthState()); +}; + +ImageReplayer<librbd::MockTestImageCtx>* ImageReplayer<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct MirrorStatusUpdater<librbd::MockTestImageCtx> { +}; + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/InstanceReplayer.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::ReturnRef; +using ::testing::WithArg; + +class TestMockInstanceReplayer : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer; + typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef MirrorStatusUpdater<librbd::MockTestImageCtx> MockMirrorStatusUpdater; + typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon; + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillOnce(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event_after(MockThreads &mock_threads, + Context** timer_ctx = nullptr) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce(DoAll( + WithArg<1>(Invoke([this, &mock_threads, timer_ctx](Context *ctx) { + ceph_assert(ceph_mutex_is_locked(mock_threads.timer_lock)); + if (timer_ctx != nullptr) { + *timer_ctx = ctx; + mock_threads.timer_cond.notify_one(); + } else { + m_threads->work_queue->queue( + new LambdaContext([&mock_threads, ctx](int) { + std::lock_guard timer_lock{mock_threads.timer_lock}; + ctx->complete(0); + }), 0); + } + })), + ReturnArg<1>())); + } + + void expect_cancel_event(MockThreads &mock_threads, bool canceled) { + EXPECT_CALL(*mock_threads.timer, cancel_event(_)) + .WillOnce(Return(canceled)); + } +}; + +TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) { + MockThreads mock_threads(m_threads); + MockServiceDaemon mock_service_daemon; + MockMirrorStatusUpdater mock_status_updater; + MockInstanceWatcher mock_instance_watcher; + MockImageReplayer mock_image_replayer; + MockInstanceReplayer instance_replayer( + m_local_io_ctx, "local_mirror_uuid", + &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr, + nullptr); + std::string global_image_id("global_image_id"); + + EXPECT_CALL(mock_image_replayer, get_global_image_id()) + .WillRepeatedly(ReturnRef(global_image_id)); + + InSequence seq; + expect_work_queue(mock_threads); + Context *timer_ctx = nullptr; + expect_add_event_after(mock_threads, &timer_ctx); + instance_replayer.init(); + instance_replayer.add_peer({"peer_uuid", m_remote_io_ctx, {}, nullptr}); + + // Acquire + + C_SaferCond on_acquire; + EXPECT_CALL(mock_image_replayer, add_peer(_)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blocklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, start(_, false)) + .WillOnce(CompleteContext(0)); + expect_work_queue(mock_threads); + + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + // Release + + C_SaferCond on_release; + + EXPECT_CALL(mock_image_replayer, is_stopped()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_running()) + .WillOnce(Return(false)); + expect_work_queue(mock_threads); + expect_add_event_after(mock_threads); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, is_stopped()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_running()) + .WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, stop(_, false)) + .WillOnce(CompleteContext(0)); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, is_stopped()) + .WillOnce(Return(true)); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, destroy()); + + instance_replayer.release_image("global_image_id", &on_release); + ASSERT_EQ(0, on_release.wait()); + + expect_work_queue(mock_threads); + expect_cancel_event(mock_threads, true); + expect_work_queue(mock_threads); + instance_replayer.shut_down(); + ASSERT_TRUE(timer_ctx != nullptr); + delete timer_ctx; +} + +TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) { + MockThreads mock_threads(m_threads); + MockServiceDaemon mock_service_daemon; + MockMirrorStatusUpdater mock_status_updater; + MockInstanceWatcher mock_instance_watcher; + MockImageReplayer mock_image_replayer; + MockInstanceReplayer instance_replayer( + m_local_io_ctx, "local_mirror_uuid", + &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr, + nullptr); + std::string global_image_id("global_image_id"); + + EXPECT_CALL(mock_image_replayer, get_global_image_id()) + .WillRepeatedly(ReturnRef(global_image_id)); + + InSequence seq; + expect_work_queue(mock_threads); + Context *timer_ctx1 = nullptr; + expect_add_event_after(mock_threads, &timer_ctx1); + instance_replayer.init(); + instance_replayer.add_peer({"peer_uuid", m_remote_io_ctx, {}, nullptr}); + + // Acquire + + C_SaferCond on_acquire; + EXPECT_CALL(mock_image_replayer, add_peer(_)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blocklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, start(_, false)) + .WillOnce(CompleteContext(0)); + expect_work_queue(mock_threads); + + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + // periodic start timer + Context *timer_ctx2 = nullptr; + expect_add_event_after(mock_threads, &timer_ctx2); + + Context *start_image_replayers_ctx = nullptr; + EXPECT_CALL(*mock_threads.work_queue, queue(_, 0)) + .WillOnce(Invoke([&start_image_replayers_ctx](Context *ctx, int r) { + start_image_replayers_ctx = ctx; + })); + + ASSERT_TRUE(timer_ctx1 != nullptr); + { + std::lock_guard timer_locker{mock_threads.timer_lock}; + timer_ctx1->complete(0); + } + + // remove finished image replayer + EXPECT_CALL(mock_image_replayer, get_health_state()).WillOnce( + Return(image_replayer::HEALTH_STATE_OK)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blocklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, destroy()); + EXPECT_CALL(mock_service_daemon, + add_or_update_namespace_attribute(_, _, _, _)).Times(3); + + ASSERT_TRUE(start_image_replayers_ctx != nullptr); + start_image_replayers_ctx->complete(0); + + // shut down + expect_work_queue(mock_threads); + expect_cancel_event(mock_threads, true); + expect_work_queue(mock_threads); + instance_replayer.shut_down(); + ASSERT_TRUE(timer_ctx2 != nullptr); + delete timer_ctx2; +} + +TEST_F(TestMockInstanceReplayer, Reacquire) { + MockThreads mock_threads(m_threads); + MockServiceDaemon mock_service_daemon; + MockMirrorStatusUpdater mock_status_updater; + MockInstanceWatcher mock_instance_watcher; + MockImageReplayer mock_image_replayer; + MockInstanceReplayer instance_replayer( + m_local_io_ctx, "local_mirror_uuid", + &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr, + nullptr); + std::string global_image_id("global_image_id"); + + EXPECT_CALL(mock_image_replayer, get_global_image_id()) + .WillRepeatedly(ReturnRef(global_image_id)); + + InSequence seq; + expect_work_queue(mock_threads); + Context *timer_ctx = nullptr; + expect_add_event_after(mock_threads, &timer_ctx); + instance_replayer.init(); + instance_replayer.add_peer({"peer_uuid", m_remote_io_ctx, {}, nullptr}); + + // Acquire + + EXPECT_CALL(mock_image_replayer, add_peer(_)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blocklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, start(_, false)) + .WillOnce(CompleteContext(0)); + expect_work_queue(mock_threads); + + C_SaferCond on_acquire1; + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire1); + ASSERT_EQ(0, on_acquire1.wait()); + + // Re-acquire + EXPECT_CALL(mock_image_replayer, set_finished(false)); + EXPECT_CALL(mock_image_replayer, restart(_)) + .WillOnce(CompleteContext(0)); + expect_work_queue(mock_threads); + + C_SaferCond on_acquire2; + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire2); + ASSERT_EQ(0, on_acquire2.wait()); + + expect_work_queue(mock_threads); + expect_cancel_event(mock_threads, true); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + expect_work_queue(mock_threads); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, destroy()); + instance_replayer.shut_down(); + ASSERT_TRUE(timer_ctx != nullptr); + delete timer_ctx; +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc new file mode 100644 index 000000000..f57654b36 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc @@ -0,0 +1,987 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librados/AioCompletionImpl.h" +#include "librbd/ManagedLock.h" +#include "test/librados/test_cxx.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "tools/rbd_mirror/InstanceReplayer.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +template <> +struct ManagedLock<MockTestImageCtx> { + static ManagedLock* s_instance; + + static ManagedLock *create(librados::IoCtx& ioctx, + librbd::AsioEngine& asio_engine, + const std::string& oid, librbd::Watcher *watcher, + managed_lock::Mode mode, + bool blocklist_on_break_lock, + uint32_t blocklist_expire_seconds) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + ManagedLock() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~ManagedLock() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD1(shut_down, void(Context *)); + MOCK_METHOD1(acquire_lock, void(Context *)); + MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *)); + MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *)); +}; + +ManagedLock<MockTestImageCtx> *ManagedLock<MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + librbd::AsioEngine* asio_engine; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue), asio_engine(threads->asio_engine) { + } +}; + +template <> +struct InstanceReplayer<librbd::MockTestImageCtx> { + MOCK_METHOD3(acquire_image, void(InstanceWatcher<librbd::MockTestImageCtx> *, + const std::string &, Context *)); + MOCK_METHOD2(release_image, void(const std::string &, Context *)); + MOCK_METHOD3(remove_peer_image, void(const std::string&, const std::string&, + Context *)); +}; + +template <> +struct Throttler<librbd::MockTestImageCtx> { + static Throttler* s_instance; + + Throttler() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + virtual ~Throttler() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD3(start_op, void(const std::string &, const std::string &, + Context *)); + MOCK_METHOD2(finish_op, void(const std::string &, const std::string &)); + MOCK_METHOD2(drain, void(const std::string &, int)); +}; + +Throttler<librbd::MockTestImageCtx>* Throttler<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/InstanceWatcher.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockInstanceWatcher : public TestMockFixture { +public: + typedef librbd::ManagedLock<librbd::MockTestImageCtx> MockManagedLock; + typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + std::string m_instance_id; + std::string m_oid; + MockThreads *m_mock_threads; + + void SetUp() override { + TestFixture::SetUp(); + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); + + m_instance_id = stringify(m_local_io_ctx.get_instance_id()); + m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id; + + m_mock_threads = new MockThreads(m_threads); + } + + void TearDown() override { + delete m_mock_threads; + TestMockFixture::TearDown(); + } + + void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { + EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _)); + } + + void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx, + const std::string &instance_id) { + std::string oid = RBD_MIRROR_INSTANCE_PREFIX + instance_id; + EXPECT_CALL(mock_io_ctx, aio_watch(oid, _, _, _)); + } + + void expect_unregister_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { + EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _)); + } + + void expect_register_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, + int r) { + EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), + StrEq("mirror_instances_add"), _, _, _, _)) + .WillOnce(Return(r)); + } + + void expect_unregister_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, + int r) { + EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), + StrEq("mirror_instances_remove"), _, _, _, _)) + .WillOnce(Return(r)); + } + + void expect_acquire_lock(MockManagedLock &mock_managed_lock, int r) { + EXPECT_CALL(mock_managed_lock, acquire_lock(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_release_lock(MockManagedLock &mock_managed_lock, int r) { + EXPECT_CALL(mock_managed_lock, shut_down(_)).WillOnce(CompleteContext(r)); + } + + void expect_destroy_lock(MockManagedLock &mock_managed_lock, + Context *ctx = nullptr) { + EXPECT_CALL(mock_managed_lock, destroy()) + .WillOnce(Invoke([ctx]() { + if (ctx != nullptr) { + ctx->complete(0); + } + })); + } + + void expect_get_locker(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r) { + EXPECT_CALL(mock_managed_lock, get_locker(_, _)) + .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out, + Context *ctx) { + if (r == 0) { + *out = locker; + } + ctx->complete(r); + })); + } + + void expect_break_lock(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r) { + EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _)) + .WillOnce(WithArg<2>(CompleteContext(r))); + } +}; + +TEST_F(TestMockInstanceWatcher, InitShutdown) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, + m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, InitError) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, + m_instance_id); + InSequence seq; + + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, -EINVAL); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + + ASSERT_EQ(-EINVAL, instance_watcher->init()); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, ShutdownError) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, + m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Shutdown + expect_release_lock(mock_managed_lock, -EINVAL); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + + +TEST_F(TestMockInstanceWatcher, Remove) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + librbd::managed_lock::Locker + locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123}; + + InSequence seq; + + expect_get_locker(mock_managed_lock, locker, 0); + expect_break_lock(mock_managed_lock, locker, 0); + expect_unregister_instance(mock_io_ctx, 0); + C_SaferCond on_destroy; + expect_destroy_lock(mock_managed_lock, &on_destroy); + + C_SaferCond on_remove; + MockInstanceWatcher::remove_instance(m_local_io_ctx, + *m_mock_threads->asio_engine, + "instance_id", &on_remove); + ASSERT_EQ(0, on_remove.wait()); + ASSERT_EQ(0, on_destroy.wait()); +} + +TEST_F(TestMockInstanceWatcher, RemoveNoent) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + InSequence seq; + + expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT); + expect_unregister_instance(mock_io_ctx, 0); + C_SaferCond on_destroy; + expect_destroy_lock(mock_managed_lock, &on_destroy); + + C_SaferCond on_remove; + MockInstanceWatcher::remove_instance(m_local_io_ctx, + *m_mock_threads->asio_engine, + "instance_id", &on_remove); + ASSERT_EQ(0, on_remove.wait()); + ASSERT_EQ(0, on_destroy.wait()); +} + +TEST_F(TestMockInstanceWatcher, ImageAcquireRelease) { + MockManagedLock mock_managed_lock; + + librados::IoCtx& io_ctx1 = m_local_io_ctx; + std::string instance_id1 = m_instance_id; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + MockInstanceReplayer mock_instance_replayer1; + auto instance_watcher1 = MockInstanceWatcher::create( + io_ctx1, *m_mock_threads->asio_engine, &mock_instance_replayer1, nullptr); + + librados::Rados cluster; + librados::IoCtx io_ctx2; + EXPECT_EQ("", connect_cluster_pp(cluster)); + EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); + std::string instance_id2 = stringify(io_ctx2.get_instance_id()); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + MockInstanceReplayer mock_instance_replayer2; + auto instance_watcher2 = MockInstanceWatcher::create( + io_ctx2, *m_mock_threads->asio_engine, &mock_instance_replayer2, nullptr); + + InSequence seq; + + // Init instance watcher 1 + expect_register_instance(mock_io_ctx1, 0); + expect_register_watch(mock_io_ctx1, instance_id1); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher1->init()); + + // Init instance watcher 2 + expect_register_instance(mock_io_ctx2, 0); + expect_register_watch(mock_io_ctx2, instance_id2); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher2->init()); + + // Acquire Image on the the same instance + EXPECT_CALL(mock_instance_replayer1, acquire_image(instance_watcher1, "gid", + _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_acquire1; + instance_watcher1->notify_image_acquire(instance_id1, "gid", &on_acquire1); + ASSERT_EQ(0, on_acquire1.wait()); + + // Acquire Image on the other instance + EXPECT_CALL(mock_instance_replayer2, acquire_image(instance_watcher2, "gid", + _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_acquire2; + instance_watcher1->notify_image_acquire(instance_id2, "gid", &on_acquire2); + ASSERT_EQ(0, on_acquire2.wait()); + + // Release Image on the the same instance + EXPECT_CALL(mock_instance_replayer1, release_image("gid", _)) + .WillOnce(WithArg<1>(CompleteContext(0))); + C_SaferCond on_release1; + instance_watcher1->notify_image_release(instance_id1, "gid", &on_release1); + ASSERT_EQ(0, on_release1.wait()); + + // Release Image on the other instance + EXPECT_CALL(mock_instance_replayer2, release_image("gid", _)) + .WillOnce(WithArg<1>(CompleteContext(0))); + C_SaferCond on_release2; + instance_watcher1->notify_image_release(instance_id2, "gid", &on_release2); + ASSERT_EQ(0, on_release2.wait()); + + // Shutdown instance watcher 1 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx1); + expect_unregister_instance(mock_io_ctx1, 0); + instance_watcher1->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher1; + + // Shutdown instance watcher 2 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx2); + expect_unregister_instance(mock_io_ctx2, 0); + instance_watcher2->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher2; +} + +TEST_F(TestMockInstanceWatcher, PeerImageRemoved) { + MockManagedLock mock_managed_lock; + + librados::IoCtx& io_ctx1 = m_local_io_ctx; + std::string instance_id1 = m_instance_id; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + MockInstanceReplayer mock_instance_replayer1; + auto instance_watcher1 = MockInstanceWatcher::create( + io_ctx1, *m_mock_threads->asio_engine, &mock_instance_replayer1, nullptr); + + librados::Rados cluster; + librados::IoCtx io_ctx2; + EXPECT_EQ("", connect_cluster_pp(cluster)); + EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); + std::string instance_id2 = stringify(io_ctx2.get_instance_id()); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + MockInstanceReplayer mock_instance_replayer2; + auto instance_watcher2 = MockInstanceWatcher::create( + io_ctx2, *m_mock_threads->asio_engine, &mock_instance_replayer2, nullptr); + + InSequence seq; + + // Init instance watcher 1 + expect_register_instance(mock_io_ctx1, 0); + expect_register_watch(mock_io_ctx1, instance_id1); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher1->init()); + + // Init instance watcher 2 + expect_register_instance(mock_io_ctx2, 0); + expect_register_watch(mock_io_ctx2, instance_id2); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher2->init()); + + // Peer Image Removed on the same instance + EXPECT_CALL(mock_instance_replayer1, remove_peer_image("gid", "uuid", _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_removed1; + instance_watcher1->notify_peer_image_removed(instance_id1, "gid", "uuid", + &on_removed1); + ASSERT_EQ(0, on_removed1.wait()); + + // Peer Image Removed on the other instance + EXPECT_CALL(mock_instance_replayer2, remove_peer_image("gid", "uuid", _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_removed2; + instance_watcher1->notify_peer_image_removed(instance_id2, "gid", "uuid", + &on_removed2); + ASSERT_EQ(0, on_removed2.wait()); + + // Shutdown instance watcher 1 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx1); + expect_unregister_instance(mock_io_ctx1, 0); + instance_watcher1->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher1; + + // Shutdown instance watcher 2 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx2); + expect_unregister_instance(mock_io_ctx2, 0); + instance_watcher2->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher2; +} + +TEST_F(TestMockInstanceWatcher, ImageAcquireReleaseCancel) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, + m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Send Acquire Image and cancel + EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) + .WillOnce(Invoke( + [this, instance_watcher, &mock_io_ctx]( + const std::string& o, librados::AioCompletionImpl *c, + bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { + c->get(); + auto ctx = new LambdaContext( + [instance_watcher, &mock_io_ctx, c, pbl](int r) { + instance_watcher->cancel_notify_requests("other"); + encode(librbd::watcher::NotifyResponse(), *pbl); + mock_io_ctx.get_mock_rados_client()-> + finish_aio_completion(c, -ETIMEDOUT); + }); + m_threads->work_queue->queue(ctx, 0); + })); + + C_SaferCond on_acquire; + instance_watcher->notify_image_acquire("other", "gid", &on_acquire); + ASSERT_EQ(-ECANCELED, on_acquire.wait()); + + // Send Release Image and cancel + EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) + .WillOnce(Invoke( + [this, instance_watcher, &mock_io_ctx]( + const std::string& o, librados::AioCompletionImpl *c, + bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { + c->get(); + auto ctx = new LambdaContext( + [instance_watcher, &mock_io_ctx, c, pbl](int r) { + instance_watcher->cancel_notify_requests("other"); + encode(librbd::watcher::NotifyResponse(), *pbl); + mock_io_ctx.get_mock_rados_client()-> + finish_aio_completion(c, -ETIMEDOUT); + }); + m_threads->work_queue->queue(ctx, 0); + })); + + C_SaferCond on_release; + instance_watcher->notify_image_release("other", "gid", &on_release); + ASSERT_EQ(-ECANCELED, on_release.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, PeerImageAcquireWatchDNE) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + MockInstanceReplayer mock_instance_replayer; + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, *m_mock_threads->asio_engine, &mock_instance_replayer, + nullptr, m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Acquire image on dead (blocklisted) instance + C_SaferCond on_acquire; + instance_watcher->notify_image_acquire("dead instance", "global image id", + &on_acquire); + ASSERT_EQ(-ENOENT, on_acquire.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, PeerImageReleaseWatchDNE) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + MockInstanceReplayer mock_instance_replayer; + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, *m_mock_threads->asio_engine, &mock_instance_replayer, + nullptr, m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Release image on dead (blocklisted) instance + C_SaferCond on_acquire; + instance_watcher->notify_image_release("dead instance", "global image id", + &on_acquire); + ASSERT_EQ(-ENOENT, on_acquire.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, PeerImageRemovedCancel) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, *m_mock_threads->asio_engine, nullptr, nullptr, + m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Send Acquire Image and cancel + EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) + .WillOnce(Invoke( + [this, instance_watcher, &mock_io_ctx]( + const std::string& o, librados::AioCompletionImpl *c, + bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { + c->get(); + auto ctx = new LambdaContext( + [instance_watcher, &mock_io_ctx, c, pbl](int r) { + instance_watcher->cancel_notify_requests("other"); + encode(librbd::watcher::NotifyResponse(), *pbl); + mock_io_ctx.get_mock_rados_client()-> + finish_aio_completion(c, -ETIMEDOUT); + }); + m_threads->work_queue->queue(ctx, 0); + })); + + C_SaferCond on_acquire; + instance_watcher->notify_peer_image_removed("other", "gid", "uuid", + &on_acquire); + ASSERT_EQ(-ECANCELED, on_acquire.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +class TestMockInstanceWatcher_NotifySync : public TestMockInstanceWatcher { +public: + typedef Throttler<librbd::MockTestImageCtx> MockThrottler; + + MockManagedLock mock_managed_lock; + MockThrottler mock_image_sync_throttler; + std::string instance_id1; + std::string instance_id2; + + librados::Rados cluster; + librados::IoCtx io_ctx2; + + MockInstanceWatcher *instance_watcher1; + MockInstanceWatcher *instance_watcher2; + + void SetUp() override { + TestMockInstanceWatcher::SetUp(); + + instance_id1 = m_instance_id; + librados::IoCtx& io_ctx1 = m_local_io_ctx; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + instance_watcher1 = MockInstanceWatcher::create(io_ctx1, + *m_mock_threads->asio_engine, + nullptr, + &mock_image_sync_throttler); + EXPECT_EQ("", connect_cluster_pp(cluster)); + EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); + instance_id2 = stringify(io_ctx2.get_instance_id()); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + instance_watcher2 = MockInstanceWatcher::create(io_ctx2, + *m_mock_threads->asio_engine, + nullptr, + &mock_image_sync_throttler); + InSequence seq; + + // Init instance watcher 1 (leader) + expect_register_instance(mock_io_ctx1, 0); + expect_register_watch(mock_io_ctx1, instance_id1); + expect_acquire_lock(mock_managed_lock, 0); + EXPECT_EQ(0, instance_watcher1->init()); + instance_watcher1->handle_acquire_leader(); + + // Init instance watcher 2 + expect_register_instance(mock_io_ctx2, 0); + expect_register_watch(mock_io_ctx2, instance_id2); + expect_acquire_lock(mock_managed_lock, 0); + EXPECT_EQ(0, instance_watcher2->init()); + instance_watcher2->handle_update_leader(instance_id1); + } + + void TearDown() override { + librados::IoCtx& io_ctx1 = m_local_io_ctx; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + + InSequence seq; + + expect_throttler_drain(); + instance_watcher1->handle_release_leader(); + + // Shutdown instance watcher 1 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx1); + expect_unregister_instance(mock_io_ctx1, 0); + instance_watcher1->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher1; + + // Shutdown instance watcher 2 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx2); + expect_unregister_instance(mock_io_ctx2, 0); + instance_watcher2->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher2; + + TestMockInstanceWatcher::TearDown(); + } + + void expect_throttler_start_op(const std::string &sync_id, + Context *on_call = nullptr, + Context **on_start_ctx = nullptr) { + EXPECT_CALL(mock_image_sync_throttler, start_op("", sync_id, _)) + .WillOnce(Invoke([on_call, on_start_ctx] (const std::string &, + const std::string &, + Context *ctx) { + if (on_start_ctx != nullptr) { + *on_start_ctx = ctx; + } else { + ctx->complete(0); + } + if (on_call != nullptr) { + on_call->complete(0); + } + })); + } + + void expect_throttler_finish_op(const std::string &sync_id, + Context *on_finish) { + EXPECT_CALL(mock_image_sync_throttler, finish_op("", "sync_id")) + .WillOnce(Invoke([on_finish](const std::string &, const std::string &) { + on_finish->complete(0); + })); + } + + void expect_throttler_drain() { + EXPECT_CALL(mock_image_sync_throttler, drain("", -ESTALE)); + } +}; + +TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher1->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + ASSERT_FALSE(instance_watcher1->cancel_sync_request("sync_id")); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher1->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnNonLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnNonLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + ASSERT_FALSE(instance_watcher2->cancel_sync_request("sync_id")); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, CancelWaitingOnNonLeader) { + InSequence seq; + + C_SaferCond on_start_op_called; + Context *on_start_ctx; + expect_throttler_start_op("sync_id", &on_start_op_called, + &on_start_ctx); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start_op_called.wait()); + + ASSERT_TRUE(instance_watcher2->cancel_sync_request("sync_id")); + // emulate watcher timeout + on_start_ctx->complete(-ETIMEDOUT); + ASSERT_EQ(-ECANCELED, on_start.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, InFlightPrevNotification) { + // start sync when previous notification is still in flight + + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start1; + instance_watcher2->notify_sync_request("sync_id", &on_start1); + ASSERT_EQ(0, on_start1.wait()); + + C_SaferCond on_start2; + EXPECT_CALL(mock_image_sync_throttler, finish_op("", "sync_id")) + .WillOnce(Invoke([this, &on_start2](const std::string &, + const std::string &) { + instance_watcher2->notify_sync_request("sync_id", &on_start2); + })); + expect_throttler_start_op("sync_id"); + instance_watcher2->notify_sync_complete("sync_id"); + + ASSERT_EQ(0, on_start2.wait()); + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, NoInFlightReleaseAcquireLeader) { + InSequence seq; + + expect_throttler_drain(); + instance_watcher1->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnLeaderReleaseLeader) { + InSequence seq; + + expect_throttler_drain(); + instance_watcher1->handle_release_leader(); + instance_watcher2->handle_acquire_leader(); + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + expect_throttler_drain(); + instance_watcher2->handle_release_leader(); + instance_watcher2->notify_sync_complete("sync_id"); + + instance_watcher1->handle_acquire_leader(); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnLeaderReleaseLeader) { + InSequence seq; + + C_SaferCond on_start_op_called; + Context *on_start_ctx; + expect_throttler_start_op("sync_id", &on_start_op_called, &on_start_ctx); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start_op_called.wait()); + + expect_throttler_drain(); + instance_watcher1->handle_release_leader(); + // emulate throttler queue drain on leader release + on_start_ctx->complete(-ESTALE); + + expect_throttler_start_op("sync_id"); + instance_watcher2->handle_acquire_leader(); + instance_watcher1->handle_update_leader(instance_id2); + + ASSERT_EQ(0, on_start.wait()); + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher1->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); + + expect_throttler_drain(); + instance_watcher2->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnNonLeaderAcquireLeader) { + InSequence seq; + + expect_throttler_drain(); + instance_watcher1->handle_release_leader(); + instance_watcher2->handle_acquire_leader(); + instance_watcher1->handle_update_leader(instance_id2); + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + expect_throttler_drain(); + instance_watcher2->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); + instance_watcher2->handle_update_leader(instance_id1); + + instance_watcher1->notify_sync_complete("sync_id"); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnNonLeaderAcquireLeader) { + InSequence seq; + + C_SaferCond on_start_op_called; + Context *on_start_ctx; + expect_throttler_start_op("sync_id", &on_start_op_called, + &on_start_ctx); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start_op_called.wait()); + + expect_throttler_drain(); + instance_watcher1->handle_release_leader(); + // emulate throttler queue drain on leader release + on_start_ctx->complete(-ESTALE); + + EXPECT_CALL(mock_image_sync_throttler, start_op("", "sync_id", _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + instance_watcher2->handle_acquire_leader(); + instance_watcher1->handle_update_leader(instance_id2); + + ASSERT_EQ(0, on_start.wait()); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); + + expect_throttler_drain(); + instance_watcher2->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_LeaderWatcher.cc b/src/test/rbd_mirror/test_mock_LeaderWatcher.cc new file mode 100644 index 000000000..d4710f9de --- /dev/null +++ b/src/test/rbd_mirror/test_mock_LeaderWatcher.cc @@ -0,0 +1,614 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/AsioEngine.h" +#include "librbd/Utils.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "tools/rbd_mirror/LeaderWatcher.h" +#include "tools/rbd_mirror/Threads.h" + +using librbd::util::create_async_context_callback; + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +struct MockManagedLock { + static MockManagedLock *s_instance; + static MockManagedLock &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + MockManagedLock() { + s_instance = this; + } + + bool m_release_lock_on_shutdown = false; + Context *m_on_released = nullptr; + + MOCK_METHOD0(construct, void()); + MOCK_METHOD0(destroy, void()); + + MOCK_CONST_METHOD0(is_lock_owner, bool()); + + MOCK_METHOD1(shut_down, void(Context *)); + MOCK_METHOD1(try_acquire_lock, void(Context *)); + MOCK_METHOD1(release_lock, void(Context *)); + MOCK_METHOD0(reacquire_lock, void()); + MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *)); + MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *)); + + MOCK_METHOD0(set_state_post_acquiring, void()); + + MOCK_CONST_METHOD0(is_shutdown, bool()); + + MOCK_CONST_METHOD0(is_state_post_acquiring, bool()); + MOCK_CONST_METHOD0(is_state_pre_releasing, bool()); + MOCK_CONST_METHOD0(is_state_locked, bool()); +}; + +MockManagedLock *MockManagedLock::s_instance = nullptr; + +template <> +struct ManagedLock<MockTestImageCtx> { + ManagedLock(librados::IoCtx& ioctx, librbd::AsioEngine& asio_engine, + const std::string& oid, librbd::Watcher *watcher, + managed_lock::Mode mode, bool blocklist_on_break_lock, + uint32_t blocklist_expire_seconds) + : m_work_queue(asio_engine.get_work_queue()) { + MockManagedLock::get_instance().construct(); + } + + virtual ~ManagedLock() { + MockManagedLock::get_instance().destroy(); + } + + librbd::asio::ContextWQ *m_work_queue; + + mutable ceph::mutex m_lock = ceph::make_mutex("ManagedLock::m_lock"); + + bool is_lock_owner() const { + return MockManagedLock::get_instance().is_lock_owner(); + } + + void shut_down(Context *on_shutdown) { + if (MockManagedLock::get_instance().m_release_lock_on_shutdown) { + on_shutdown = new LambdaContext( + [this, on_shutdown](int r) { + MockManagedLock::get_instance().m_release_lock_on_shutdown = false; + shut_down(on_shutdown); + }); + release_lock(on_shutdown); + return; + } + + MockManagedLock::get_instance().shut_down(on_shutdown); + } + + void try_acquire_lock(Context *on_acquired) { + Context *post_acquire_ctx = create_async_context_callback( + m_work_queue, new LambdaContext( + [this, on_acquired](int r) { + post_acquire_lock_handler(r, on_acquired); + })); + MockManagedLock::get_instance().try_acquire_lock(post_acquire_ctx); + } + + void release_lock(Context *on_released) { + ceph_assert(MockManagedLock::get_instance().m_on_released == nullptr); + MockManagedLock::get_instance().m_on_released = on_released; + + Context *post_release_ctx = new LambdaContext( + [this](int r) { + ceph_assert(MockManagedLock::get_instance().m_on_released != nullptr); + post_release_lock_handler(false, r, + MockManagedLock::get_instance().m_on_released); + MockManagedLock::get_instance().m_on_released = nullptr; + }); + + Context *release_ctx = new LambdaContext( + [post_release_ctx](int r) { + if (r < 0) { + MockManagedLock::get_instance().m_on_released->complete(r); + } else { + MockManagedLock::get_instance().release_lock(post_release_ctx); + } + }); + + Context *pre_release_ctx = new LambdaContext( + [this, release_ctx](int r) { + bool shutting_down = + MockManagedLock::get_instance().m_release_lock_on_shutdown; + pre_release_lock_handler(shutting_down, release_ctx); + }); + + m_work_queue->queue(pre_release_ctx, 0); + } + + void reacquire_lock(Context* on_finish) { + MockManagedLock::get_instance().reacquire_lock(); + } + + void get_locker(managed_lock::Locker *locker, Context *on_finish) { + MockManagedLock::get_instance().get_locker(locker, on_finish); + } + + void break_lock(const managed_lock::Locker &locker, bool force_break_lock, + Context *on_finish) { + MockManagedLock::get_instance().break_lock(locker, force_break_lock, + on_finish); + } + + void set_state_post_acquiring() { + MockManagedLock::get_instance().set_state_post_acquiring(); + } + + bool is_shutdown() const { + return MockManagedLock::get_instance().is_shutdown(); + } + + bool is_state_post_acquiring() const { + return MockManagedLock::get_instance().is_state_post_acquiring(); + } + + bool is_state_pre_releasing() const { + return MockManagedLock::get_instance().is_state_pre_releasing(); + } + + bool is_state_locked() const { + return MockManagedLock::get_instance().is_state_locked(); + } + + virtual void post_acquire_lock_handler(int r, Context *on_finish) = 0; + virtual void pre_release_lock_handler(bool shutting_down, + Context *on_finish) = 0; + virtual void post_release_lock_handler(bool shutting_down, int r, + Context *on_finish) = 0; +}; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + librbd::AsioEngine* asio_engine; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue), asio_engine(threads->asio_engine) { + } +}; + +template <> +struct Instances<librbd::MockTestImageCtx> { + static Instances* s_instance; + + static Instances *create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &ioctx, + const std::string& instance_id, + instances::Listener&) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + Instances() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~Instances() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD1(init, void(Context *)); + MOCK_METHOD1(shut_down, void(Context *)); + MOCK_METHOD1(acked, void(const std::vector<std::string> &)); + MOCK_METHOD0(unblock_listener, void()); +}; + +Instances<librbd::MockTestImageCtx> *Instances<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +} // namespace rbd + + +// template definitions +#include "tools/rbd_mirror/LeaderWatcher.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; + +using librbd::MockManagedLock; + +struct MockListener : public leader_watcher::Listener { + static MockListener* s_instance; + + MockListener() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~MockListener() override { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD1(post_acquire_handler, void(Context *)); + MOCK_METHOD1(pre_release_handler, void(Context *)); + + MOCK_METHOD1(update_leader_handler, void(const std::string &)); + MOCK_METHOD1(handle_instances_added, void(const InstanceIds&)); + MOCK_METHOD1(handle_instances_removed, void(const InstanceIds&)); +}; + +MockListener *MockListener::s_instance = nullptr; + +class TestMockLeaderWatcher : public TestMockFixture { +public: + typedef Instances<librbd::MockTestImageCtx> MockInstances; + typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + void SetUp() override { + TestMockFixture::SetUp(); + m_mock_threads = new MockThreads(m_threads); + } + + void TearDown() override { + delete m_mock_threads; + TestMockFixture::TearDown(); + } + + void expect_construct(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, construct()); + } + + void expect_destroy(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, destroy()); + } + + void expect_is_lock_owner(MockManagedLock &mock_managed_lock, bool owner) { + EXPECT_CALL(mock_managed_lock, is_lock_owner()) + .WillOnce(Return(owner)); + } + + void expect_shut_down(MockManagedLock &mock_managed_lock, + bool release_lock_on_shutdown, int r) { + mock_managed_lock.m_release_lock_on_shutdown = release_lock_on_shutdown; + EXPECT_CALL(mock_managed_lock, shut_down(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_try_acquire_lock(MockManagedLock &mock_managed_lock, int r) { + EXPECT_CALL(mock_managed_lock, try_acquire_lock(_)) + .WillOnce(CompleteContext(r)); + if (r == 0) { + expect_set_state_post_acquiring(mock_managed_lock); + } + } + + void expect_release_lock(MockManagedLock &mock_managed_lock, int r, + Context *on_finish = nullptr) { + EXPECT_CALL(mock_managed_lock, release_lock(_)) + .WillOnce(Invoke([on_finish, &mock_managed_lock, r](Context *ctx) { + if (on_finish != nullptr) { + auto on_released = mock_managed_lock.m_on_released; + ceph_assert(on_released != nullptr); + mock_managed_lock.m_on_released = new LambdaContext( + [on_released, on_finish](int r) { + on_released->complete(r); + on_finish->complete(r); + }); + } + ctx->complete(r); + })); + } + + void expect_get_locker(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r) { + EXPECT_CALL(mock_managed_lock, get_locker(_, _)) + .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out, + Context *ctx) { + if (r == 0) { + *out = locker; + } + ctx->complete(r); + })); + } + + void expect_break_lock(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r, + Context *on_finish) { + EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _)) + .WillOnce(Invoke([on_finish, r](const librbd::managed_lock::Locker &, + bool, Context *ctx) { + ctx->complete(r); + on_finish->complete(0); + })); + } + + void expect_set_state_post_acquiring(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, set_state_post_acquiring()); + } + + void expect_is_shutdown(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, is_shutdown()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + } + + void expect_is_leader(MockManagedLock &mock_managed_lock, bool post_acquiring, + bool locked) { + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .WillOnce(Return(post_acquiring)); + if (!post_acquiring) { + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .WillOnce(Return(locked)); + } + } + + void expect_is_leader(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_pre_releasing()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + } + + void expect_notify_heartbeat(MockManagedLock &mock_managed_lock, + Context *on_finish) { + // is_leader in notify_heartbeat + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .WillOnce(Return(true)); + + // is_leader in handle_notify_heartbeat + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .WillOnce(DoAll(Invoke([on_finish]() { + on_finish->complete(0); + }), + Return(true))); + } + + void expect_destroy(MockInstances &mock_instances) { + EXPECT_CALL(mock_instances, destroy()); + } + + void expect_init(MockInstances &mock_instances, int r) { + EXPECT_CALL(mock_instances, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_shut_down(MockInstances &mock_instances, int r) { + EXPECT_CALL(mock_instances, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + expect_destroy(mock_instances); + } + + void expect_acquire_notify(MockManagedLock &mock_managed_lock, + MockListener &mock_listener, int r) { + expect_is_leader(mock_managed_lock, true, false); + EXPECT_CALL(mock_listener, post_acquire_handler(_)) + .WillOnce(CompleteContext(r)); + expect_is_leader(mock_managed_lock, true, false); + } + + void expect_release_notify(MockManagedLock &mock_managed_lock, + MockListener &mock_listener, int r) { + expect_is_leader(mock_managed_lock, false, false); + EXPECT_CALL(mock_listener, pre_release_handler(_)) + .WillOnce(CompleteContext(r)); + expect_is_leader(mock_managed_lock, false, false); + } + + void expect_unblock_listener(MockInstances& mock_instances) { + EXPECT_CALL(mock_instances, unblock_listener()); + } + + void expect_instances_acked(MockInstances& mock_instances) { + EXPECT_CALL(mock_instances, acked(_)); + } + + MockThreads *m_mock_threads; +}; + +TEST_F(TestMockLeaderWatcher, InitShutdown) { + MockManagedLock mock_managed_lock; + MockInstances mock_instances; + MockListener listener; + + expect_is_shutdown(mock_managed_lock); + expect_destroy(mock_managed_lock); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + C_SaferCond on_heartbeat_finish; + expect_is_leader(mock_managed_lock, false, false); + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Shutdown + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + expect_release_lock(mock_managed_lock, 0); + expect_shut_down(mock_managed_lock, true, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +TEST_F(TestMockLeaderWatcher, InitReleaseShutdown) { + MockManagedLock mock_managed_lock; + MockInstances mock_instances; + MockListener listener; + + expect_is_shutdown(mock_managed_lock); + expect_destroy(mock_managed_lock); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + C_SaferCond on_heartbeat_finish; + expect_is_leader(mock_managed_lock, false, false); + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Release + expect_is_leader(mock_managed_lock, false, true); + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + C_SaferCond on_release; + expect_release_lock(mock_managed_lock, 0, &on_release); + + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release.wait()); + + // Shutdown + expect_shut_down(mock_managed_lock, false, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +TEST_F(TestMockLeaderWatcher, AcquireError) { + MockManagedLock mock_managed_lock; + MockInstances mock_instances; + MockListener listener; + + expect_is_shutdown(mock_managed_lock); + expect_is_leader(mock_managed_lock); + expect_destroy(mock_managed_lock); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + C_SaferCond on_heartbeat_finish; + expect_is_leader(mock_managed_lock, false, false); + expect_try_acquire_lock(mock_managed_lock, -EAGAIN); + expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT); + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Shutdown + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + expect_release_lock(mock_managed_lock, 0); + expect_shut_down(mock_managed_lock, true, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +TEST_F(TestMockLeaderWatcher, Break) { + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", "1")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_missed_heartbeats", + "1")); + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + int max_acquire_attempts = cct->_conf.get_val<uint64_t>( + "rbd_mirror_leader_max_acquire_attempts_before_break"); + + MockManagedLock mock_managed_lock; + MockInstances mock_instances; + MockListener listener; + librbd::managed_lock::Locker + locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123}; + + expect_is_shutdown(mock_managed_lock); + expect_is_leader(mock_managed_lock); + expect_destroy(mock_managed_lock); + EXPECT_CALL(listener, update_leader_handler(_)); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + expect_is_leader(mock_managed_lock, false, false); + for (int i = 0; i < max_acquire_attempts; i++) { + expect_try_acquire_lock(mock_managed_lock, -EAGAIN); + expect_get_locker(mock_managed_lock, locker, 0); + } + C_SaferCond on_break; + expect_break_lock(mock_managed_lock, locker, 0, &on_break); + C_SaferCond on_heartbeat_finish; + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Shutdown + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + expect_release_lock(mock_managed_lock, 0); + expect_shut_down(mock_managed_lock, true, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc new file mode 100644 index 000000000..4db576eb4 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc @@ -0,0 +1,706 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/stringify.h" +#include "tools/rbd_mirror/MirrorStatusUpdater.h" +#include "tools/rbd_mirror/MirrorStatusWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include <map> +#include <string> +#include <utility> + +namespace librbd { +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct MirrorStatusWatcher<librbd::MockTestImageCtx> { + static MirrorStatusWatcher* s_instance; + static MirrorStatusWatcher* create(librados::IoCtx& io_ctx, + MockContextWQ* mock_context_wq) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + MirrorStatusWatcher() { + s_instance = this; + } +}; + +MirrorStatusWatcher<librbd::MockTestImageCtx>* MirrorStatusWatcher<librbd::MockTestImageCtx>::s_instance = nullptr; + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/MirrorStatusUpdater.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoDefault; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::StrEq; +using ::testing::Return; +using ::testing::WithArg; + +class TestMockMirrorStatusUpdater : public TestMockFixture { +public: + typedef MirrorStatusUpdater<librbd::MockTestImageCtx> MockMirrorStatusUpdater; + typedef MirrorStatusWatcher<librbd::MockTestImageCtx> MockMirrorStatusWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + typedef std::map<std::string, cls::rbd::MirrorImageSiteStatus> + MirrorImageSiteStatuses; + + void SetUp() override { + TestMockFixture::SetUp(); + + m_mock_local_io_ctx = &get_mock_io_ctx(m_local_io_ctx); + m_mock_threads = new MockThreads(m_threads); + } + + void TearDown() override { + delete m_mock_threads; + TestMockFixture::TearDown(); + } + + void expect_timer_add_event(Context** timer_event) { + EXPECT_CALL(*m_mock_threads->timer, add_event_after(_, _)) + .WillOnce(WithArg<1>(Invoke([timer_event](Context *ctx) { + *timer_event = ctx; + return ctx; + }))); + } + + void expect_timer_cancel_event() { + EXPECT_CALL(*m_mock_threads->timer, cancel_event(_)) + .WillOnce(Invoke([](Context* ctx) { + delete ctx; + return false; + })); + } + + void expect_work_queue(bool async) { + EXPECT_CALL(*m_mock_threads->work_queue, queue(_, _)) + .WillOnce(Invoke([this, async](Context *ctx, int r) { + if (async) { + m_threads->work_queue->queue(ctx, r); + } else { + ctx->complete(r); + } + })); + } + + void expect_mirror_status_watcher_init( + MockMirrorStatusWatcher& mock_mirror_status_watcher, int r) { + EXPECT_CALL(*mock_mirror_status_watcher.s_instance, init(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_mirror_status_watcher_shut_down( + MockMirrorStatusWatcher& mock_mirror_status_watcher, int r) { + EXPECT_CALL(*mock_mirror_status_watcher.s_instance, shut_down(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_mirror_status_update( + const std::string& global_image_id, + const cls::rbd::MirrorImageSiteStatus& mirror_image_status, int r) { + EXPECT_CALL(*m_mock_local_io_ctx, + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_status_set"), _, _, _, _)) + .WillOnce(WithArg<4>(Invoke( + [r, global_image_id, mirror_image_status](bufferlist& in_bl) { + auto bl_it = in_bl.cbegin(); + std::string decode_global_image_id; + decode(decode_global_image_id, bl_it); + EXPECT_EQ(global_image_id, decode_global_image_id); + + cls::rbd::MirrorImageSiteStatus decode_mirror_image_status; + decode(decode_mirror_image_status, bl_it); + EXPECT_EQ(mirror_image_status, decode_mirror_image_status); + return r; + }))); + } + + void expect_mirror_status_update( + const MirrorImageSiteStatuses& mirror_image_site_statuses, + const std::string& mirror_uuid, int r) { + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke([this](auto&&... args) { + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + return r; + })); + + for (auto [global_image_id, mirror_image_status] : + mirror_image_site_statuses) { + mirror_image_status.mirror_uuid = mirror_uuid; + expect_mirror_status_update(global_image_id, mirror_image_status, r); + if (r < 0) { + break; + } + } + } + + void expect_mirror_status_remove(const std::string& global_image_id, int r) { + EXPECT_CALL(*m_mock_local_io_ctx, + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_status_remove"), _, _, _, _)) + .WillOnce(WithArg<4>(Invoke( + [r, global_image_id](bufferlist& in_bl) { + auto bl_it = in_bl.cbegin(); + std::string decode_global_image_id; + decode(decode_global_image_id, bl_it); + EXPECT_EQ(global_image_id, decode_global_image_id); + + return r; + }))); + } + + void expect_mirror_status_removes(const std::set<std::string>& mirror_images, + int r) { + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke([this](auto&&... args) { + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + return r; + })); + + for (auto global_image_id : mirror_images) { + expect_mirror_status_remove(global_image_id, r); + if (r < 0) { + break; + } + } + } + + void fire_timer_event(Context** timer_event, + Context** update_task) { + expect_timer_add_event(timer_event); + + // timer queues the update task + EXPECT_CALL(*m_mock_threads->work_queue, queue(_, _)) + .WillOnce(WithArg<0>(Invoke([update_task](Context* ctx) mutable { + *update_task = ctx; + }))); + + // fire the timer task + { + std::lock_guard timer_locker{m_mock_threads->timer_lock}; + ceph_assert(*timer_event != nullptr); + (*timer_event)->complete(0); + } + } + + void init_mirror_status_updater( + MockMirrorStatusUpdater& mock_mirror_status_updater, + MockMirrorStatusWatcher& mock_mirror_status_watcher, + Context** timer_event) { + expect_timer_add_event(timer_event); + expect_mirror_status_watcher_init(mock_mirror_status_watcher, 0); + expect_work_queue(true); + + C_SaferCond ctx; + mock_mirror_status_updater.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + } + + void shut_down_mirror_status_updater( + MockMirrorStatusUpdater& mock_mirror_status_updater, + MockMirrorStatusWatcher& mock_mirror_status_watcher) { + expect_timer_cancel_event(); + expect_mirror_status_watcher_shut_down(mock_mirror_status_watcher, 0); + expect_work_queue(true); + + C_SaferCond ctx; + mock_mirror_status_updater.shut_down(&ctx); + ASSERT_EQ(0, ctx.wait()); + } + + librados::MockTestMemIoCtxImpl* m_mock_local_io_ctx = nullptr; + MockThreads* m_mock_threads = nullptr; +}; + +TEST_F(TestMockMirrorStatusUpdater, InitShutDown) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, InitStatusWatcherError) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + Context* timer_event = nullptr; + expect_timer_add_event(&timer_event); + expect_mirror_status_watcher_init(*mock_mirror_status_watcher, -EINVAL); + expect_timer_cancel_event(); + expect_work_queue(true); + + C_SaferCond ctx; + mock_mirror_status_updater.init(&ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockMirrorStatusUpdater, ShutDownStatusWatcherError) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + C_SaferCond on_shutdown; + expect_timer_cancel_event(); + expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher, -EINVAL); + expect_work_queue(true); + mock_mirror_status_updater.shut_down(&on_shutdown); + + ASSERT_EQ(-EINVAL, on_shutdown.wait()); +} + +TEST_F(TestMockMirrorStatusUpdater, SmallBatch) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + MirrorImageSiteStatuses mirror_image_site_statuses; + for (auto i = 0; i < 100; ++i) { + auto pair = mirror_image_site_statuses.emplace( + stringify(i), cls::rbd::MirrorImageSiteStatus{}); + mock_mirror_status_updater.set_mirror_image_status(pair.first->first, + pair.first->second, + false); + } + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + expect_mirror_status_update(mirror_image_site_statuses, "", 0); + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, LargeBatch) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + MirrorImageSiteStatuses mirror_image_site_statuses; + for (auto i = 0; i < 200; ++i) { + auto pair = mirror_image_site_statuses.emplace( + stringify(i), cls::rbd::MirrorImageSiteStatus{}); + mock_mirror_status_updater.set_mirror_image_status(pair.first->first, + pair.first->second, + false); + } + + auto it_1 = mirror_image_site_statuses.begin(); + auto it_2 = mirror_image_site_statuses.begin(); + std::advance(it_2, 100); + MirrorImageSiteStatuses mirror_image_site_statuses_1{it_1, it_2}; + + it_1 = it_2; + std::advance(it_2, 100); + MirrorImageSiteStatuses mirror_image_site_statuses_2{it_1, it_2}; + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + expect_mirror_status_update(mirror_image_site_statuses_1, "", 0); + expect_mirror_status_update(mirror_image_site_statuses_2, "", 0); + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + mock_mirror_status_updater.set_mirror_image_status( + "1", {"", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}, + false); + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + expect_mirror_status_update( + {{"1", cls::rbd::MirrorImageSiteStatus{ + "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, + "", 0); + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, RemoveStatus) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + C_SaferCond ctx; + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + expect_work_queue(false); + mock_mirror_status_updater.remove_mirror_image_status("1", false, &ctx); + ASSERT_EQ(0, ctx.wait()); + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + C_SaferCond remove_flush_ctx; + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke([this, &remove_flush_ctx](auto&&... args) { + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + remove_flush_ctx.complete(r); + return r; + })); + expect_mirror_status_remove("1", 0); + update_task->complete(0); + ASSERT_EQ(0, remove_flush_ctx.wait()); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, OverwriteRemoveStatus) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + C_SaferCond ctx; + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + expect_work_queue(false); + mock_mirror_status_updater.remove_mirror_image_status("1", false, &ctx); + ASSERT_EQ(0, ctx.wait()); + mock_mirror_status_updater.set_mirror_image_status( + "1", {"", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}, + false); + + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + expect_mirror_status_update( + {{"1", cls::rbd::MirrorImageSiteStatus{ + "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, + "", 0); + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke([this, &mock_mirror_status_updater](auto&&... args) { + mock_mirror_status_updater.set_mirror_image_status( + "1", {"", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, + "description"}, + true); + + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + return r; + })); + expect_mirror_status_update("1", cls::rbd::MirrorImageSiteStatus{}, 0); + expect_work_queue(false); + expect_mirror_status_update( + {{"1", cls::rbd::MirrorImageSiteStatus{ + "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, + "", 0); + + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + expect_work_queue(false); + expect_mirror_status_update({{"1", cls::rbd::MirrorImageSiteStatus{}}}, + "", 0); + mock_mirror_status_updater.set_mirror_image_status("1", {}, true); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, RemoveImmediateUpdate) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + + C_SaferCond ctx; + expect_work_queue(false); + expect_mirror_status_removes({"1"}, 0); + expect_work_queue(false); + mock_mirror_status_updater.remove_mirror_image_status("1", true, &ctx); + ASSERT_EQ(0, ctx.wait()); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, RemoveRefreshIdleStatus) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + + C_SaferCond ctx; + expect_work_queue(true); + mock_mirror_status_updater.remove_refresh_mirror_image_status("1", &ctx); + ASSERT_EQ(0, ctx.wait()); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, RemoveRefreshInFlightStatus) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + C_SaferCond on_removed; + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke( + [this, &mock_mirror_status_updater, &on_removed](auto&&... args) { + mock_mirror_status_updater.remove_refresh_mirror_image_status( + "1", &on_removed); + + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + return r; + })); + update_task->complete(0); + ASSERT_EQ(0, on_removed.wait()); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, ShutDownWhileUpdating) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + C_SaferCond on_shutdown; + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke( + [this, &mock_mirror_status_updater, &on_shutdown](auto&&... args) { + mock_mirror_status_updater.shut_down(&on_shutdown); + m_threads->work_queue->drain(); + + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + return r; + })); + + expect_timer_cancel_event(); + expect_mirror_status_watcher_shut_down(*mock_mirror_status_watcher, 0); + + update_task->complete(0); + ASSERT_EQ(0, on_shutdown.wait()); +} + +TEST_F(TestMockMirrorStatusUpdater, MirrorPeerSitePing) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, + "mirror uuid"); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + MirrorImageSiteStatuses mirror_image_site_statuses; + for (auto i = 0; i < 100; ++i) { + auto pair = mirror_image_site_statuses.emplace( + stringify(i), cls::rbd::MirrorImageSiteStatus{}); + mock_mirror_status_updater.set_mirror_image_status(pair.first->first, + pair.first->second, + false); + } + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + expect_mirror_status_update(mirror_image_site_statuses, "mirror uuid", 0); + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc new file mode 100644 index 000000000..ece1a3396 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -0,0 +1,611 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/api/Config.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "tools/rbd_mirror/NamespaceReplayer.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/ImageMap.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/InstanceReplayer.h" +#include "tools/rbd_mirror/MirrorStatusUpdater.h" +#include "tools/rbd_mirror/PoolWatcher.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct ImageDeleter<librbd::MockTestImageCtx> { + static ImageDeleter* s_instance; + + static ImageDeleter* create( + librados::IoCtx &ioctx, Threads<librbd::MockTestImageCtx> *threads, + Throttler<librbd::MockTestImageCtx> *image_deletion_throttler, + ServiceDaemon<librbd::MockTestImageCtx> *service_daemon) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*)); + + ImageDeleter() { + s_instance = this; + } +}; + +ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct ImageMap<librbd::MockTestImageCtx> { + static ImageMap* s_instance; + + static ImageMap *create(librados::IoCtx &ioctx, + Threads<librbd::MockTestImageCtx> *threads, + const std::string& instance_id, + image_map::Listener &listener) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + MOCK_METHOD1(update_instances_added, void(const std::vector<std::string>&)); + MOCK_METHOD1(update_instances_removed, void(const std::vector<std::string>&)); + + MOCK_METHOD3(update_images_mock, void(const std::string&, + const std::set<std::string>&, + const std::set<std::string>&)); + void update_images(const std::string& mirror_uuid, + std::set<std::string>&& added, + std::set<std::string>&& removed) { + update_images_mock(mirror_uuid, added, removed); + } + + ImageMap() { + s_instance = this; + } +}; + +ImageMap<librbd::MockTestImageCtx>* ImageMap<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct InstanceReplayer<librbd::MockTestImageCtx> { + static InstanceReplayer* s_instance; + + static InstanceReplayer* create( + librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid, + Threads<librbd::MockTestImageCtx> *threads, + ServiceDaemon<librbd::MockTestImageCtx> *service_daemon, + MirrorStatusUpdater<librbd::MockTestImageCtx>* local_status_updater, + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD0(start, void()); + MOCK_METHOD0(stop, void()); + MOCK_METHOD0(restart, void()); + MOCK_METHOD0(flush, void()); + + MOCK_METHOD1(stop, void(Context *)); + + MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*)); + + MOCK_METHOD1(add_peer, void(const Peer<librbd::MockTestImageCtx>&)); + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + MOCK_METHOD1(release_all, void(Context*)); + + InstanceReplayer() { + s_instance = this; + } +}; + +InstanceReplayer<librbd::MockTestImageCtx>* InstanceReplayer<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { + static InstanceWatcher* s_instance; + + static InstanceWatcher* create( + librados::IoCtx &ioctx, librbd::AsioEngine& asio_engine, + InstanceReplayer<librbd::MockTestImageCtx>* instance_replayer, + Throttler<librbd::MockTestImageCtx> *image_sync_throttler) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD0(handle_acquire_leader, void()); + MOCK_METHOD0(handle_release_leader, void()); + + MOCK_METHOD0(get_instance_id, std::string()); + + MOCK_METHOD2(print_sync_status, void(Formatter*, std::stringstream*)); + + MOCK_METHOD1(init, void(Context *)); + MOCK_METHOD1(shut_down, void(Context *)); + + MOCK_METHOD3(notify_image_acquire, void(const std::string&, + const std::string&, + Context*)); + MOCK_METHOD3(notify_image_release, void(const std::string&, + const std::string&, + Context*)); + MOCK_METHOD4(notify_peer_image_removed, void(const std::string&, + const std::string&, + const std::string&, + Context*)); + + MOCK_METHOD1(handle_update_leader, void(const std::string&)); + + InstanceWatcher() { + s_instance = this; + } + +}; + +InstanceWatcher<librbd::MockTestImageCtx>* InstanceWatcher<librbd::MockTestImageCtx>::s_instance = nullptr; + +template <> +struct MirrorStatusUpdater<librbd::MockTestImageCtx> { + std::string local_mirror_uuid; + + static std::map<std::string, MirrorStatusUpdater*> s_instance; + + static MirrorStatusUpdater *create(librados::IoCtx &io_ctx, + Threads<librbd::MockTestImageCtx> *threads, + const std::string& local_mirror_uuid) { + ceph_assert(s_instance[local_mirror_uuid] != nullptr); + return s_instance[local_mirror_uuid]; + } + + MirrorStatusUpdater(const std::string_view& local_mirror_uuid) + : local_mirror_uuid(local_mirror_uuid) { + s_instance[std::string{local_mirror_uuid}] = this; + } + ~MirrorStatusUpdater() { + s_instance.erase(local_mirror_uuid); + } + + MOCK_METHOD1(init, void(Context *)); + MOCK_METHOD1(shut_down, void(Context *)); +}; + +std::map<std::string, MirrorStatusUpdater<librbd::MockTestImageCtx> *> + MirrorStatusUpdater<librbd::MockTestImageCtx>::s_instance; + +template<> +struct PoolWatcher<librbd::MockTestImageCtx> { + int64_t pool_id = -1; + + static std::map<int64_t, PoolWatcher *> s_instances; + + static PoolWatcher *create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &ioctx, + const std::string& mirror_uuid, + pool_watcher::Listener& listener) { + auto pool_id = ioctx.get_id(); + ceph_assert(s_instances.count(pool_id)); + return s_instances[pool_id]; + } + + MOCK_METHOD0(is_blocklisted, bool()); + + MOCK_METHOD0(get_image_count, uint64_t()); + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + PoolWatcher(int64_t pool_id) : pool_id(pool_id) { + ceph_assert(!s_instances.count(pool_id)); + s_instances[pool_id] = this; + } + ~PoolWatcher() { + s_instances.erase(pool_id); + } +}; + +std::map<int64_t, PoolWatcher<librbd::MockTestImageCtx> *> PoolWatcher<librbd::MockTestImageCtx>::s_instances; + +template<> +struct ServiceDaemon<librbd::MockTestImageCtx> { + MOCK_METHOD4(add_or_update_namespace_attribute, + void(int64_t, const std::string&, const std::string&, + const service_daemon::AttributeValue&)); + MOCK_METHOD2(remove_attribute, + void(int64_t, const std::string&)); + + MOCK_METHOD4(add_or_update_callout, uint64_t(int64_t, uint64_t, + service_daemon::CalloutLevel, + const std::string&)); + MOCK_METHOD2(remove_callout, void(int64_t, uint64_t)); +}; + +template <> +struct Threads<librbd::MockTestImageCtx> { + ceph::mutex &timer_lock; + SafeTimer *timer; + librbd::asio::ContextWQ *work_queue; + librbd::AsioEngine* asio_engine; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue), asio_engine(threads->asio_engine) { + } +}; + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/NamespaceReplayer.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockNamespaceReplayer : public TestMockFixture { +public: + typedef NamespaceReplayer<librbd::MockTestImageCtx> MockNamespaceReplayer; + typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter; + typedef ImageMap<librbd::MockTestImageCtx> MockImageMap; + typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef MirrorStatusUpdater<librbd::MockTestImageCtx> MockMirrorStatusUpdater; + typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher; + typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + void SetUp() override { + TestMockFixture::SetUp(); + m_mock_threads = new MockThreads(m_threads); + } + + void TearDown() override { + delete m_mock_threads; + TestMockFixture::TearDown(); + } + + void expect_mirror_status_updater_init( + MockMirrorStatusUpdater &mock_mirror_status_updater, int r) { + EXPECT_CALL(mock_mirror_status_updater, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_mirror_status_updater_shut_down( + MockMirrorStatusUpdater &mock_mirror_status_updater) { + EXPECT_CALL(mock_mirror_status_updater, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + void expect_instance_replayer_init( + MockInstanceReplayer& mock_instance_replayer, int r) { + EXPECT_CALL(mock_instance_replayer, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_instance_replayer_shut_down( + MockInstanceReplayer& mock_instance_replayer) { + EXPECT_CALL(mock_instance_replayer, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + void expect_instance_replayer_stop( + MockInstanceReplayer& mock_instance_replayer) { + EXPECT_CALL(mock_instance_replayer, stop(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + void expect_instance_replayer_add_peer( + MockInstanceReplayer& mock_instance_replayer) { + EXPECT_CALL(mock_instance_replayer, add_peer(_)); + } + + void expect_instance_replayer_release_all( + MockInstanceReplayer& mock_instance_replayer) { + EXPECT_CALL(mock_instance_replayer, release_all(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + void expect_instance_watcher_get_instance_id( + MockInstanceWatcher& mock_instance_watcher, + const std::string &instance_id) { + EXPECT_CALL(mock_instance_watcher, get_instance_id()) + .WillOnce(Return(instance_id)); + } + + void expect_instance_watcher_init( + MockInstanceWatcher& mock_instance_watcher, int r) { + EXPECT_CALL(mock_instance_watcher, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_instance_watcher_shut_down( + MockInstanceWatcher& mock_instance_watcher) { + EXPECT_CALL(mock_instance_watcher, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + void expect_instance_watcher_handle_acquire_leader( + MockInstanceWatcher& mock_instance_watcher) { + EXPECT_CALL(mock_instance_watcher, handle_acquire_leader()); + } + + void expect_instance_watcher_handle_release_leader( + MockInstanceWatcher& mock_instance_watcher) { + EXPECT_CALL(mock_instance_watcher, handle_release_leader()); + } + + void expect_image_map_init(MockInstanceWatcher &mock_instance_watcher, + MockImageMap& mock_image_map, int r) { + expect_instance_watcher_get_instance_id(mock_instance_watcher, "1234"); + EXPECT_CALL(mock_image_map, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_image_map_shut_down(MockImageMap& mock_image_map) { + EXPECT_CALL(mock_image_map, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + void expect_pool_watcher_init(MockPoolWatcher& mock_pool_watcher, int r) { + EXPECT_CALL(mock_pool_watcher, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_pool_watcher_shut_down(MockPoolWatcher& mock_pool_watcher) { + EXPECT_CALL(mock_pool_watcher, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + void expect_image_deleter_init(MockImageDeleter& mock_image_deleter, int r) { + EXPECT_CALL(mock_image_deleter, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_image_deleter_shut_down(MockImageDeleter& mock_image_deleter) { + EXPECT_CALL(mock_image_deleter, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, 0)); + } + + MockThreads *m_mock_threads; +}; + +TEST_F(TestMockNamespaceReplayer, Init_LocalMirrorStatusUpdaterError) { + InSequence seq; + + auto mock_local_mirror_status_updater = new MockMirrorStatusUpdater{""}; + expect_mirror_status_updater_init(*mock_local_mirror_status_updater, -EINVAL); + + MockNamespaceReplayer namespace_replayer( + {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, + nullptr, nullptr, nullptr, nullptr, nullptr); + + C_SaferCond on_init; + namespace_replayer.init(&on_init); + ASSERT_EQ(-EINVAL, on_init.wait()); +} + +TEST_F(TestMockNamespaceReplayer, Init_RemoteMirrorStatusUpdaterError) { + InSequence seq; + + auto mock_local_mirror_status_updater = new MockMirrorStatusUpdater{""}; + expect_mirror_status_updater_init(*mock_local_mirror_status_updater, 0); + + auto mock_remote_mirror_status_updater = new MockMirrorStatusUpdater{ + "local mirror uuid"}; + expect_mirror_status_updater_init(*mock_remote_mirror_status_updater, + -EINVAL); + + expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); + + MockNamespaceReplayer namespace_replayer( + {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, + nullptr, nullptr, nullptr, nullptr, nullptr); + + C_SaferCond on_init; + namespace_replayer.init(&on_init); + ASSERT_EQ(-EINVAL, on_init.wait()); +} + +TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) { + InSequence seq; + + auto mock_local_mirror_status_updater = new MockMirrorStatusUpdater{""}; + expect_mirror_status_updater_init(*mock_local_mirror_status_updater, 0); + + auto mock_remote_mirror_status_updater = new MockMirrorStatusUpdater{ + "local mirror uuid"}; + expect_mirror_status_updater_init(*mock_remote_mirror_status_updater, 0); + + auto mock_instance_replayer = new MockInstanceReplayer(); + expect_instance_replayer_init(*mock_instance_replayer, -EINVAL); + + expect_mirror_status_updater_shut_down(*mock_remote_mirror_status_updater); + expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); + + MockNamespaceReplayer namespace_replayer( + {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, + nullptr, nullptr, nullptr, nullptr, nullptr); + + C_SaferCond on_init; + namespace_replayer.init(&on_init); + ASSERT_EQ(-EINVAL, on_init.wait()); +} + +TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) { + InSequence seq; + + auto mock_local_mirror_status_updater = new MockMirrorStatusUpdater{""}; + expect_mirror_status_updater_init(*mock_local_mirror_status_updater, 0); + + auto mock_remote_mirror_status_updater = new MockMirrorStatusUpdater{ + "local mirror uuid"}; + expect_mirror_status_updater_init(*mock_remote_mirror_status_updater, 0); + + auto mock_instance_replayer = new MockInstanceReplayer(); + expect_instance_replayer_init(*mock_instance_replayer, 0); + expect_instance_replayer_add_peer(*mock_instance_replayer); + + auto mock_instance_watcher = new MockInstanceWatcher(); + expect_instance_watcher_init(*mock_instance_watcher, -EINVAL); + + expect_instance_replayer_shut_down(*mock_instance_replayer); + expect_mirror_status_updater_shut_down(*mock_remote_mirror_status_updater); + expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); + + MockNamespaceReplayer namespace_replayer( + {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, + nullptr, nullptr, nullptr, nullptr, nullptr); + + C_SaferCond on_init; + namespace_replayer.init(&on_init); + ASSERT_EQ(-EINVAL, on_init.wait()); +} + +TEST_F(TestMockNamespaceReplayer, Init) { + InSequence seq; + + auto mock_local_mirror_status_updater = new MockMirrorStatusUpdater{""}; + expect_mirror_status_updater_init(*mock_local_mirror_status_updater, 0); + + auto mock_remote_mirror_status_updater = new MockMirrorStatusUpdater{ + "local mirror uuid"}; + expect_mirror_status_updater_init(*mock_remote_mirror_status_updater, 0); + + auto mock_instance_replayer = new MockInstanceReplayer(); + expect_instance_replayer_init(*mock_instance_replayer, 0); + expect_instance_replayer_add_peer(*mock_instance_replayer); + + auto mock_instance_watcher = new MockInstanceWatcher(); + expect_instance_watcher_init(*mock_instance_watcher, 0); + + MockServiceDaemon mock_service_daemon; + MockNamespaceReplayer namespace_replayer( + {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, + nullptr, nullptr, &mock_service_daemon, nullptr, nullptr); + + C_SaferCond on_init; + namespace_replayer.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + expect_instance_replayer_stop(*mock_instance_replayer); + expect_instance_watcher_shut_down(*mock_instance_watcher); + expect_instance_replayer_shut_down(*mock_instance_replayer); + expect_mirror_status_updater_shut_down(*mock_remote_mirror_status_updater); + expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); + + C_SaferCond on_shut_down; + namespace_replayer.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} + +TEST_F(TestMockNamespaceReplayer, AcquireLeader) { + InSequence seq; + + // init + + auto mock_local_mirror_status_updater = new MockMirrorStatusUpdater{""}; + expect_mirror_status_updater_init(*mock_local_mirror_status_updater, 0); + + auto mock_remote_mirror_status_updater = new MockMirrorStatusUpdater{ + "local mirror uuid"}; + expect_mirror_status_updater_init(*mock_remote_mirror_status_updater, 0); + + auto mock_instance_replayer = new MockInstanceReplayer(); + expect_instance_replayer_init(*mock_instance_replayer, 0); + expect_instance_replayer_add_peer(*mock_instance_replayer); + + auto mock_instance_watcher = new MockInstanceWatcher(); + expect_instance_watcher_init(*mock_instance_watcher, 0); + + MockServiceDaemon mock_service_daemon; + MockNamespaceReplayer namespace_replayer( + {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", + "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, + nullptr, nullptr, &mock_service_daemon, nullptr, nullptr); + + C_SaferCond on_init; + namespace_replayer.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + // acquire leader + + expect_instance_watcher_handle_acquire_leader(*mock_instance_watcher); + + auto mock_image_map = new MockImageMap(); + expect_image_map_init(*mock_instance_watcher, *mock_image_map, 0); + + auto mock_local_pool_watcher = new MockPoolWatcher(m_local_io_ctx.get_id()); + expect_pool_watcher_init(*mock_local_pool_watcher, 0); + + auto mock_remote_pool_watcher = new MockPoolWatcher(m_remote_io_ctx.get_id()); + expect_pool_watcher_init(*mock_remote_pool_watcher, 0); + + auto mock_image_deleter = new MockImageDeleter(); + expect_image_deleter_init(*mock_image_deleter, 0); + + C_SaferCond on_acquire; + namespace_replayer.handle_acquire_leader(&on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + // release leader + + expect_instance_watcher_handle_release_leader(*mock_instance_watcher); + expect_image_deleter_shut_down(*mock_image_deleter); + expect_pool_watcher_shut_down(*mock_local_pool_watcher); + expect_pool_watcher_shut_down(*mock_remote_pool_watcher); + expect_image_map_shut_down(*mock_image_map); + expect_instance_replayer_release_all(*mock_instance_replayer); + + // shut down + + expect_instance_replayer_stop(*mock_instance_replayer); + expect_instance_watcher_shut_down(*mock_instance_watcher); + expect_instance_replayer_shut_down(*mock_instance_replayer); + expect_mirror_status_updater_shut_down(*mock_remote_mirror_status_updater); + expect_mirror_status_updater_shut_down(*mock_local_mirror_status_updater); + + C_SaferCond on_shut_down; + namespace_replayer.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc new file mode 100644 index 000000000..ebd27d7e1 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -0,0 +1,934 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/api/Config.h" +#include "librbd/api/Namespace.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librados_test_stub/MockTestMemCluster.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "tools/rbd_mirror/Throttler.h" +#include "tools/rbd_mirror/LeaderWatcher.h" +#include "tools/rbd_mirror/NamespaceReplayer.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/PoolReplayer.h" +#include "tools/rbd_mirror/RemotePoolPoller.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" +#include "common/Formatter.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace api { + +template <> +class Config<MockTestImageCtx> { +public: + static void apply_pool_overrides(librados::IoCtx& io_ctx, + ConfigProxy* config_proxy) { + } +}; + +template <> +class Namespace<MockTestImageCtx> { +public: + static Namespace* s_instance; + + static int list(librados::IoCtx& io_ctx, std::vector<std::string> *names) { + if (s_instance) { + return s_instance->list(names); + } + + return 0; + } + + Namespace() { + s_instance = this; + } + + void add(const std::string &name) { + std::lock_guard locker{m_lock}; + + m_names.insert(name); + } + + void remove(const std::string &name) { + std::lock_guard locker{m_lock}; + + m_names.erase(name); + } + + void clear() { + std::lock_guard locker{m_lock}; + + m_names.clear(); + } + +private: + ceph::mutex m_lock = ceph::make_mutex("Namespace"); + std::set<std::string> m_names; + + int list(std::vector<std::string> *names) { + std::lock_guard locker{m_lock}; + + names->clear(); + names->insert(names->begin(), m_names.begin(), m_names.end()); + return 0; + } +}; + +Namespace<librbd::MockTestImageCtx>* Namespace<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace api + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Throttler<librbd::MockTestImageCtx> { + static Throttler* s_instance; + + static Throttler *create( + CephContext *cct, + const std::string &max_concurrent_ops_config_param_name) { + return s_instance; + } + + Throttler() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + virtual ~Throttler() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD1(print_status, void(Formatter*)); +}; + +Throttler<librbd::MockTestImageCtx>* Throttler<librbd::MockTestImageCtx>::s_instance = nullptr; + +template <> +struct NamespaceReplayer<librbd::MockTestImageCtx> { + static std::map<std::string, NamespaceReplayer *> s_instances; + + static NamespaceReplayer *create( + const std::string &name, + librados::IoCtx &local_ioctx, + librados::IoCtx &remote_ioctx, + const std::string &local_mirror_uuid, + const std::string& local_mirror_peer_uuid, + const RemotePoolMeta& remote_pool_meta, + Threads<librbd::MockTestImageCtx> *threads, + Throttler<librbd::MockTestImageCtx> *image_sync_throttler, + Throttler<librbd::MockTestImageCtx> *image_deletion_throttler, + ServiceDaemon<librbd::MockTestImageCtx> *service_daemon, + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { + ceph_assert(s_instances.count(name)); + auto namespace_replayer = s_instances[name]; + s_instances.erase(name); + return namespace_replayer; + } + + MOCK_METHOD0(is_blocklisted, bool()); + MOCK_METHOD0(get_instance_id, std::string()); + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + MOCK_METHOD1(handle_acquire_leader, void(Context *)); + MOCK_METHOD1(handle_release_leader, void(Context *)); + MOCK_METHOD1(handle_update_leader, void(const std::string &)); + MOCK_METHOD1(handle_instances_added, void(const std::vector<std::string> &)); + MOCK_METHOD1(handle_instances_removed, void(const std::vector<std::string> &)); + + MOCK_METHOD1(print_status, void(Formatter*)); + MOCK_METHOD0(start, void()); + MOCK_METHOD0(stop, void()); + MOCK_METHOD0(restart, void()); + MOCK_METHOD0(flush, void()); + + NamespaceReplayer(const std::string &name = "") { + ceph_assert(!s_instances.count(name)); + s_instances[name] = this; + } +}; + +std::map<std::string, NamespaceReplayer<librbd::MockTestImageCtx> *> NamespaceReplayer<librbd::MockTestImageCtx>::s_instances; + +template<> +struct LeaderWatcher<librbd::MockTestImageCtx> { + static LeaderWatcher* s_instance; + leader_watcher::Listener* listener = nullptr; + + static LeaderWatcher *create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &ioctx, + leader_watcher::Listener* listener) { + ceph_assert(s_instance != nullptr); + s_instance->listener = listener; + return s_instance; + } + + MOCK_METHOD0(is_blocklisted, bool()); + MOCK_METHOD0(is_leader, bool()); + MOCK_METHOD0(release_leader, void()); + + MOCK_METHOD1(get_leader_instance_id, bool(std::string*)); + MOCK_METHOD1(list_instances, void(std::vector<std::string>*)); + + MOCK_METHOD0(init, int()); + MOCK_METHOD0(shut_down, int()); + + LeaderWatcher() { + s_instance = this; + } + +}; + +LeaderWatcher<librbd::MockTestImageCtx>* LeaderWatcher<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct RemotePoolPoller<librbd::MockTestImageCtx> { + static RemotePoolPoller* s_instance; + + remote_pool_poller::Listener* listener = nullptr; + + static RemotePoolPoller* create( + Threads<librbd::MockTestImageCtx>* threads, + librados::IoCtx& remote_io_ctx, + const std::string& local_site_name, + const std::string& local_mirror_uuid, + remote_pool_poller::Listener& listener) { + ceph_assert(s_instance != nullptr); + s_instance->listener = &listener; + return s_instance; + } + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + RemotePoolPoller() { + s_instance = this; + } +}; + +RemotePoolPoller<librbd::MockTestImageCtx>* RemotePoolPoller<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct ServiceDaemon<librbd::MockTestImageCtx> { + MOCK_METHOD2(add_namespace, void(int64_t, const std::string &)); + MOCK_METHOD2(remove_namespace, void(int64_t, const std::string &)); + + MOCK_METHOD3(add_or_update_attribute, + void(int64_t, const std::string&, + const service_daemon::AttributeValue&)); + MOCK_METHOD2(remove_attribute, + void(int64_t, const std::string&)); + + MOCK_METHOD4(add_or_update_callout, uint64_t(int64_t, uint64_t, + service_daemon::CalloutLevel, + const std::string&)); + MOCK_METHOD2(remove_callout, void(int64_t, uint64_t)); +}; + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + ceph::condition_variable timer_cond; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/PoolReplayer.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockPoolReplayer : public TestMockFixture { +public: + typedef librbd::api::Namespace<librbd::MockTestImageCtx> MockNamespace; + typedef PoolReplayer<librbd::MockTestImageCtx> MockPoolReplayer; + typedef Throttler<librbd::MockTestImageCtx> MockThrottler; + typedef NamespaceReplayer<librbd::MockTestImageCtx> MockNamespaceReplayer; + typedef RemotePoolPoller<librbd::MockTestImageCtx> MockRemotePoolPoller; + typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher; + typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_connect(librados::MockTestMemCluster& mock_cluster, + librados::MockTestMemRadosClient* mock_rados_client, + const std::string& cluster_name, CephContext** cct_ref) { + EXPECT_CALL(mock_cluster, create_rados_client(_)) + .WillOnce(Invoke([cluster_name, mock_rados_client, cct_ref](CephContext* cct) { + EXPECT_EQ(cluster_name, cct->_conf->cluster); + if (cct_ref != nullptr) { + cct->get(); + *cct_ref = cct; + } + return mock_rados_client; + })); + } + + void expect_create_ioctx(librados::MockTestMemRadosClient* mock_rados_client, + librados::MockTestMemIoCtxImpl* mock_io_ctx_impl) { + EXPECT_CALL(*mock_rados_client, create_ioctx(_, _)) + .WillOnce(Invoke([mock_io_ctx_impl](int64_t id, const std::string& name) { + return mock_io_ctx_impl; + })); + } + + void expect_mirror_uuid_get(librados::MockTestMemIoCtxImpl *io_ctx_impl, + const std::string &uuid, int r) { + bufferlist out_bl; + encode(uuid, out_bl); + + EXPECT_CALL(*io_ctx_impl, + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), + _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + + void expect_mirror_mode_get(librados::MockTestMemIoCtxImpl *io_ctx_impl, + cls::rbd::MirrorMode mirror_mode, int r) { + bufferlist out_bl; + encode(mirror_mode, out_bl); + + EXPECT_CALL(*io_ctx_impl, + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_mode_get"), + _, _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + + void expect_mirror_mode_get(librados::MockTestMemIoCtxImpl *io_ctx_impl) { + EXPECT_CALL(*io_ctx_impl, + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_mode_get"), + _, _, _, _)) + .WillRepeatedly(DoAll(WithArg<5>(Invoke([](bufferlist *bl) { + encode(cls::rbd::MIRROR_MODE_POOL, *bl); + })), + Return(0))); + } + + void expect_leader_watcher_init(MockLeaderWatcher& mock_leader_watcher, + int r) { + EXPECT_CALL(mock_leader_watcher, init()) + .WillOnce(Return(r)); + } + + void expect_leader_watcher_shut_down(MockLeaderWatcher& mock_leader_watcher) { + EXPECT_CALL(mock_leader_watcher, shut_down()); + } + + void expect_leader_watcher_get_leader_instance_id( + MockLeaderWatcher& mock_leader_watcher) { + EXPECT_CALL(mock_leader_watcher, get_leader_instance_id(_)) + .WillRepeatedly(Return(true)); + } + + void expect_leader_watcher_list_instances( + MockLeaderWatcher& mock_leader_watcher) { + EXPECT_CALL(mock_leader_watcher, list_instances(_)) + .Times(AtLeast(0)); + } + + void expect_remote_pool_poller_init( + MockRemotePoolPoller& mock_remote_pool_poller, + const RemotePoolMeta& remote_pool_meta, int r) { + EXPECT_CALL(mock_remote_pool_poller, init(_)) + .WillOnce(Invoke( + [this, &mock_remote_pool_poller, remote_pool_meta, r] + (Context* ctx) { + if (r >= 0) { + mock_remote_pool_poller.listener->handle_updated( + remote_pool_meta); + } + + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_remote_pool_poller_shut_down( + MockRemotePoolPoller& mock_remote_pool_poller, int r) { + EXPECT_CALL(mock_remote_pool_poller, shut_down(_)) + .WillOnce(Invoke( + [this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_leader_watcher_is_blocklisted( + MockLeaderWatcher &mock_leader_watcher, bool blocklisted) { + EXPECT_CALL(mock_leader_watcher, is_blocklisted()) + .WillRepeatedly(Return(blocklisted)); + } + + void expect_namespace_replayer_is_blocklisted( + MockNamespaceReplayer &mock_namespace_replayer, + bool blocklisted) { + EXPECT_CALL(mock_namespace_replayer, is_blocklisted()) + .WillRepeatedly(Return(blocklisted)); + } + + void expect_namespace_replayer_get_instance_id( + MockNamespaceReplayer &mock_namespace_replayer, + const std::string &instance_id) { + EXPECT_CALL(mock_namespace_replayer, get_instance_id()) + .WillOnce(Return(instance_id)); + } + + void expect_namespace_replayer_init( + MockNamespaceReplayer &mock_namespace_replayer, int r, + Context *on_init = nullptr) { + + EXPECT_CALL(mock_namespace_replayer, init(_)) + .WillOnce(Invoke([this, r, on_init](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + if (on_init != nullptr) { + m_threads->work_queue->queue(on_init, r); + } + })); + } + + void expect_namespace_replayer_shut_down( + MockNamespaceReplayer &mock_namespace_replayer, + Context *on_shut_down = nullptr) { + EXPECT_CALL(mock_namespace_replayer, shut_down(_)) + .WillOnce(Invoke([this, on_shut_down](Context* ctx) { + m_threads->work_queue->queue(ctx); + if (on_shut_down != nullptr) { + m_threads->work_queue->queue(on_shut_down); + } + })); + } + + void expect_namespace_replayer_handle_acquire_leader( + MockNamespaceReplayer &mock_namespace_replayer, int r, + Context *on_acquire = nullptr) { + EXPECT_CALL(mock_namespace_replayer, handle_acquire_leader(_)) + .WillOnce(Invoke([this, r, on_acquire](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + if (on_acquire != nullptr) { + m_threads->work_queue->queue(on_acquire, r); + } + })); + } + + void expect_namespace_replayer_handle_release_leader( + MockNamespaceReplayer &mock_namespace_replayer, int r, + Context *on_release = nullptr) { + EXPECT_CALL(mock_namespace_replayer, handle_release_leader(_)) + .WillOnce(Invoke([this, r, on_release](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + if (on_release != nullptr) { + m_threads->work_queue->queue(on_release, r); + } + })); + } + + void expect_namespace_replayer_handle_update_leader( + MockNamespaceReplayer &mock_namespace_replayer, + const std::string &leader_instance_id, + Context *on_update = nullptr) { + EXPECT_CALL(mock_namespace_replayer, + handle_update_leader(leader_instance_id)) + .WillOnce(Invoke([on_update](const std::string &) { + if (on_update != nullptr) { + on_update->complete(0); + } + })); + } + + void expect_namespace_replayer_handle_instances_added( + MockNamespaceReplayer &mock_namespace_replayer) { + EXPECT_CALL(mock_namespace_replayer, handle_instances_added(_)); + } + + void expect_namespace_replayer_handle_instances_removed( + MockNamespaceReplayer &mock_namespace_replayer) { + EXPECT_CALL(mock_namespace_replayer, handle_instances_removed(_)); + } + + void expect_service_daemon_add_namespace( + MockServiceDaemon &mock_service_daemon, + const std::string& namespace_name) { + EXPECT_CALL(mock_service_daemon, + add_namespace(m_local_io_ctx.get_id(), namespace_name)); + } + + void expect_service_daemon_remove_namespace( + MockServiceDaemon &mock_service_daemon, + const std::string& namespace_name) { + EXPECT_CALL(mock_service_daemon, + remove_namespace(m_local_io_ctx.get_id(), namespace_name)); + } + + void expect_service_daemon_add_or_update_attribute( + MockServiceDaemon &mock_service_daemon, const std::string& key, + const service_daemon::AttributeValue& value) { + EXPECT_CALL(mock_service_daemon, add_or_update_attribute(_, key, value)); + } + + void expect_service_daemon_remove_attribute( + MockServiceDaemon &mock_service_daemon, const std::string& key) { + EXPECT_CALL(mock_service_daemon, remove_attribute(_, key)); + } + + void expect_service_daemon_add_or_update_instance_id_attribute( + MockServiceDaemon &mock_service_daemon, const std::string &instance_id) { + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, "instance_id", {instance_id}); + } + + PoolMetaCache m_pool_meta_cache{g_ceph_context}; +}; + +TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { + PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; + peer_spec.mon_host = "123"; + peer_spec.key = "234"; + + auto mock_default_namespace_replayer = new MockNamespaceReplayer(); + expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, + false); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); + expect_leader_watcher_is_blocklisted(*mock_leader_watcher, false); + + InSequence seq; + + auto& mock_cluster = get_mock_cluster(); + auto mock_local_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr); + + auto mock_remote_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + CephContext* remote_cct = nullptr; + expect_connect(mock_cluster, mock_remote_rados_client, "cluster name", + &remote_cct); + + auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx( + m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name()); + expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); + + expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_leader_watcher_init(*mock_leader_watcher, 0); + + MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, + m_local_io_ctx.get_id(), peer_spec, {}); + pool_replayer.init("siteA"); + + ASSERT_TRUE(remote_cct != nullptr); + ASSERT_EQ("123", remote_cct->_conf.get_val<std::string>("mon_host")); + ASSERT_EQ("234", remote_cct->_conf.get_val<std::string>("key")); + remote_cct->put(); + + expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); + + pool_replayer.shut_down(); +} + +TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { + PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; + peer_spec.mon_host = "123"; + peer_spec.key = "234"; + + auto mock_default_namespace_replayer = new MockNamespaceReplayer(); + expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, + false); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); + expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blocklisted(*mock_leader_watcher, false); + + InSequence seq; + + auto& mock_cluster = get_mock_cluster(); + auto mock_local_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr); + + auto mock_remote_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + expect_connect(mock_cluster, mock_remote_rados_client, "cluster name", + nullptr); + + auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx( + m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name()); + expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); + + expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_leader_watcher_init(*mock_leader_watcher, 0); + + MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, + m_local_io_ctx.get_id(), peer_spec, {}); + pool_replayer.init("siteA"); + + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); + expect_namespace_replayer_handle_acquire_leader( + *mock_default_namespace_replayer, 0); + + C_SaferCond on_acquire; + mock_leader_watcher->listener->post_acquire_handler(&on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + expect_service_daemon_remove_attribute(mock_service_daemon, + SERVICE_DAEMON_LEADER_KEY); + expect_namespace_replayer_handle_release_leader( + *mock_default_namespace_replayer, 0); + + C_SaferCond on_release; + mock_leader_watcher->listener->pre_release_handler(&on_release); + ASSERT_EQ(0, on_release.wait()); + + expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); + + pool_replayer.shut_down(); +} + +TEST_F(TestMockPoolReplayer, Namespaces) { + PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; + peer_spec.mon_host = "123"; + peer_spec.key = "234"; + + g_ceph_context->_conf.set_val( + "rbd_mirror_pool_replayers_refresh_interval", "1"); + + MockNamespace mock_namespace; + + auto mock_default_namespace_replayer = new MockNamespaceReplayer(); + expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, + false); + + auto mock_ns1_namespace_replayer = new MockNamespaceReplayer("ns1"); + expect_namespace_replayer_is_blocklisted(*mock_ns1_namespace_replayer, + false); + + auto mock_ns2_namespace_replayer = new MockNamespaceReplayer("ns2"); + expect_namespace_replayer_is_blocklisted(*mock_ns2_namespace_replayer, + false); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); + expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blocklisted(*mock_leader_watcher, false); + + auto& mock_cluster = get_mock_cluster(); + auto mock_local_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx( + m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name()); + auto mock_remote_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + + expect_mirror_mode_get(mock_local_io_ctx); + + InSequence seq; + + expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr); + expect_connect(mock_cluster, mock_remote_rados_client, "cluster name", + nullptr); + expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); + expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_leader_watcher_init(*mock_leader_watcher, 0); + + MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, + m_local_io_ctx.get_id(), peer_spec, {}); + pool_replayer.init("siteA"); + + C_SaferCond on_ns1_init; + expect_namespace_replayer_init(*mock_ns1_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns1"); + expect_namespace_replayer_handle_update_leader(*mock_ns1_namespace_replayer, + "", &on_ns1_init); + + mock_namespace.add("ns1"); + ASSERT_EQ(0, on_ns1_init.wait()); + + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); + expect_namespace_replayer_handle_acquire_leader( + *mock_default_namespace_replayer, 0); + expect_namespace_replayer_handle_acquire_leader( + *mock_ns1_namespace_replayer, 0); + + C_SaferCond on_acquire; + mock_leader_watcher->listener->post_acquire_handler(&on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns2"); + C_SaferCond on_ns2_acquire; + expect_namespace_replayer_handle_acquire_leader( + *mock_ns2_namespace_replayer, 0, &on_ns2_acquire); + expect_namespace_replayer_handle_instances_added( + *mock_ns2_namespace_replayer); + + mock_namespace.add("ns2"); + ASSERT_EQ(0, on_ns2_acquire.wait()); + + C_SaferCond on_ns2_shut_down; + expect_service_daemon_remove_namespace(mock_service_daemon, "ns2"); + expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer, + &on_ns2_shut_down); + mock_namespace.remove("ns2"); + ASSERT_EQ(0, on_ns2_shut_down.wait()); + + expect_service_daemon_remove_attribute(mock_service_daemon, + SERVICE_DAEMON_LEADER_KEY); + expect_namespace_replayer_handle_release_leader( + *mock_default_namespace_replayer, 0); + expect_namespace_replayer_handle_release_leader( + *mock_ns1_namespace_replayer, 0); + + C_SaferCond on_release; + mock_leader_watcher->listener->pre_release_handler(&on_release); + ASSERT_EQ(0, on_release.wait()); + + expect_service_daemon_remove_namespace(mock_service_daemon, "ns1"); + expect_namespace_replayer_shut_down(*mock_ns1_namespace_replayer); + expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); + + pool_replayer.shut_down(); +} + +TEST_F(TestMockPoolReplayer, NamespacesError) { + PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; + peer_spec.mon_host = "123"; + peer_spec.key = "234"; + + g_ceph_context->_conf.set_val( + "rbd_mirror_pool_replayers_refresh_interval", "1"); + + MockNamespace mock_namespace; + + auto mock_default_namespace_replayer = new MockNamespaceReplayer(); + expect_namespace_replayer_is_blocklisted(*mock_default_namespace_replayer, + false); + auto mock_ns1_namespace_replayer = new MockNamespaceReplayer("ns1"); + auto mock_ns2_namespace_replayer = new MockNamespaceReplayer("ns2"); + expect_namespace_replayer_is_blocklisted(*mock_ns2_namespace_replayer, + false); + auto mock_ns3_namespace_replayer = new MockNamespaceReplayer("ns3"); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_get_leader_instance_id(*mock_leader_watcher); + expect_leader_watcher_list_instances(*mock_leader_watcher); + expect_leader_watcher_is_blocklisted(*mock_leader_watcher, false); + + auto& mock_cluster = get_mock_cluster(); + auto mock_local_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx( + m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name()); + auto mock_remote_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + + expect_mirror_mode_get(mock_local_io_ctx); + + InSequence seq; + + expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr); + expect_connect(mock_cluster, mock_remote_rados_client, "cluster name", + nullptr); + expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); + expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); + expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); + expect_leader_watcher_init(*mock_leader_watcher, 0); + + MockServiceDaemon mock_service_daemon; + std::string instance_id = stringify(mock_local_io_ctx->get_instance_id()); + expect_service_daemon_add_or_update_instance_id_attribute( + mock_service_daemon, instance_id); + + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, + m_local_io_ctx.get_id(), peer_spec, {}); + pool_replayer.init("siteA"); + + // test namespace replayer init fails for non leader + + C_SaferCond on_ns1_init; + Context* ctx = new LambdaContext( + [&mock_namespace, &on_ns1_init](int r) { + mock_namespace.remove("ns1"); + on_ns1_init.complete(r); + }); + expect_namespace_replayer_init(*mock_ns1_namespace_replayer, -EINVAL, ctx); + mock_namespace.add("ns1"); + ASSERT_EQ(-EINVAL, on_ns1_init.wait()); + + // test acquire leader fails when default namespace replayer fails + + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); + expect_namespace_replayer_handle_acquire_leader( + *mock_default_namespace_replayer, -EINVAL); + + C_SaferCond on_acquire1; + mock_leader_watcher->listener->post_acquire_handler(&on_acquire1); + ASSERT_EQ(-EINVAL, on_acquire1.wait()); + + // test acquire leader succeeds when non-default namespace replayer fails + + C_SaferCond on_ns2_init; + expect_namespace_replayer_init(*mock_ns2_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns2"); + expect_namespace_replayer_handle_update_leader(*mock_ns2_namespace_replayer, + "", &on_ns2_init); + mock_namespace.add("ns2"); + ASSERT_EQ(0, on_ns2_init.wait()); + + expect_service_daemon_add_or_update_attribute( + mock_service_daemon, SERVICE_DAEMON_LEADER_KEY, true); + expect_namespace_replayer_handle_acquire_leader( + *mock_default_namespace_replayer, 0); + + expect_namespace_replayer_handle_acquire_leader(*mock_ns2_namespace_replayer, + -EINVAL); + ctx = new LambdaContext( + [&mock_namespace](int) { + mock_namespace.remove("ns2"); + }); + expect_service_daemon_remove_namespace(mock_service_daemon, "ns2"); + expect_namespace_replayer_shut_down(*mock_ns2_namespace_replayer, ctx); + mock_namespace.add("ns2"); + + C_SaferCond on_acquire2; + mock_leader_watcher->listener->post_acquire_handler(&on_acquire2); + ASSERT_EQ(0, on_acquire2.wait()); + + // test namespace replayer init fails on acquire leader + + C_SaferCond on_ns3_shut_down; + ctx = new LambdaContext( + [&mock_namespace, &on_ns3_shut_down](int) { + mock_namespace.remove("ns3"); + on_ns3_shut_down.complete(0); + }); + expect_namespace_replayer_init(*mock_ns3_namespace_replayer, 0); + expect_service_daemon_add_namespace(mock_service_daemon, "ns3"); + expect_namespace_replayer_handle_acquire_leader(*mock_ns3_namespace_replayer, + -EINVAL); + expect_service_daemon_remove_namespace(mock_service_daemon, "ns3"); + expect_namespace_replayer_shut_down(*mock_ns3_namespace_replayer, ctx); + mock_namespace.add("ns3"); + ASSERT_EQ(0, on_ns3_shut_down.wait()); + + expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); + + pool_replayer.shut_down(); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc new file mode 100644 index 000000000..44efc0532 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc @@ -0,0 +1,728 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "librbd/MirroringWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/PoolWatcher.h" +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h" +#include "include/stringify.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +struct MockMirroringWatcher { + static MockMirroringWatcher *s_instance; + static MockMirroringWatcher &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + MockMirroringWatcher() { + s_instance = this; + } + + MOCK_CONST_METHOD0(is_unregistered, bool()); + MOCK_METHOD1(register_watch, void(Context*)); + MOCK_METHOD1(unregister_watch, void(Context*)); + + MOCK_CONST_METHOD0(get_oid, std::string()); +}; + +template <> +struct MirroringWatcher<MockTestImageCtx> { + static MirroringWatcher *s_instance; + + MirroringWatcher(librados::IoCtx &io_ctx, ::MockContextWQ *work_queue) { + s_instance = this; + } + virtual ~MirroringWatcher() { + } + + static MirroringWatcher<MockTestImageCtx> &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + virtual void handle_rewatch_complete(int r) = 0; + + virtual void handle_mode_updated(cls::rbd::MirrorMode mirror_mode) = 0; + virtual void handle_image_updated(cls::rbd::MirrorImageState state, + const std::string &remote_image_id, + const std::string &global_image_id) = 0; + + bool is_unregistered() const { + return MockMirroringWatcher::get_instance().is_unregistered(); + } + void register_watch(Context *ctx) { + MockMirroringWatcher::get_instance().register_watch(ctx); + } + void unregister_watch(Context *ctx) { + MockMirroringWatcher::get_instance().unregister_watch(ctx); + } + std::string get_oid() const { + return MockMirroringWatcher::get_instance().get_oid(); + } +}; + +MockMirroringWatcher *MockMirroringWatcher::s_instance = nullptr; +MirroringWatcher<MockTestImageCtx> *MirroringWatcher<MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + ceph::mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +namespace pool_watcher { + +template <> +struct RefreshImagesRequest<librbd::MockTestImageCtx> { + ImageIds *image_ids = nullptr; + Context *on_finish = nullptr; + static RefreshImagesRequest *s_instance; + static RefreshImagesRequest *create(librados::IoCtx &io_ctx, + ImageIds *image_ids, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ids = image_ids; + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + RefreshImagesRequest() { + s_instance = this; + } +}; + +RefreshImagesRequest<librbd::MockTestImageCtx> *RefreshImagesRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace pool_watcher + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/PoolWatcher.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithoutArgs; + +class TestMockPoolWatcher : public TestMockFixture { +public: + typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef pool_watcher::RefreshImagesRequest<librbd::MockTestImageCtx> MockRefreshImagesRequest; + typedef librbd::MockMirroringWatcher MockMirroringWatcher; + typedef librbd::MirroringWatcher<librbd::MockTestImageCtx> MirroringWatcher; + + struct MockListener : pool_watcher::Listener { + TestMockPoolWatcher *test; + + MockListener(TestMockPoolWatcher *test) : test(test) { + } + + MOCK_METHOD3(mock_handle_update, void(const std::string &, const ImageIds &, + const ImageIds &)); + void handle_update(const std::string &mirror_uuid, + ImageIds &&added_image_ids, + ImageIds &&removed_image_ids) override { + mock_handle_update(mirror_uuid, added_image_ids, removed_image_ids); + } + }; + + TestMockPoolWatcher() = default; + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_mirroring_watcher_is_unregistered(MockMirroringWatcher &mock_mirroring_watcher, + bool unregistered) { + EXPECT_CALL(mock_mirroring_watcher, is_unregistered()) + .WillOnce(Return(unregistered)); + } + + void expect_mirroring_watcher_register(MockMirroringWatcher &mock_mirroring_watcher, + int r) { + EXPECT_CALL(mock_mirroring_watcher, register_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_mirroring_watcher_unregister(MockMirroringWatcher &mock_mirroring_watcher, + int r) { + EXPECT_CALL(mock_mirroring_watcher, unregister_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_refresh_images(MockRefreshImagesRequest &request, + const ImageIds &image_ids, int r) { + EXPECT_CALL(request, send()) + .WillOnce(Invoke([&request, image_ids, r]() { + *request.image_ids = image_ids; + request.on_finish->complete(r); + })); + } + + void expect_listener_handle_update(MockListener &mock_listener, + const std::string &mirror_uuid, + const ImageIds &added_image_ids, + const ImageIds &removed_image_ids) { + EXPECT_CALL(mock_listener, mock_handle_update(mirror_uuid, added_image_ids, + removed_image_ids)) + .WillOnce(WithoutArgs(Invoke([this]() { + std::lock_guard locker{m_lock}; + ++m_update_count; + m_cond.notify_all(); + }))); + } + + void expect_timer_add_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = + new LambdaContext([this, ctx](int r) { + std::lock_guard timer_locker{m_threads->timer_lock}; + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), + ReturnArg<1>())); + } + + int when_shut_down(MockPoolWatcher &mock_pool_watcher) { + C_SaferCond ctx; + mock_pool_watcher.shut_down(&ctx); + return ctx.wait(); + } + + bool wait_for_update(uint32_t count) { + std::unique_lock locker{m_lock}; + if (m_cond.wait_for(locker, 10s, + [count, this] { return m_update_count >= count; })) { + m_update_count -= count; + return true; + } else { + return false; + } + } + + ceph::mutex m_lock = ceph::make_mutex("TestMockPoolWatcher::m_lock"); + ceph::condition_variable m_cond; + uint32_t m_update_count = 0; +}; + +TEST_F(TestMockPoolWatcher, EmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, NonEmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1"}, + {"global id 2", "remote id 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, image_ids, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1"}, + {"global id 2", "remote id 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + bool refresh_sent = false; + EXPECT_CALL(mock_refresh_images_request, send()) + .WillOnce(Invoke([this, &mock_refresh_images_request, &image_ids, + &refresh_sent]() { + *mock_refresh_images_request.image_ids = image_ids; + + std::lock_guard locker{m_lock}; + refresh_sent = true; + m_cond.notify_all(); + })); + + + MockListener mock_listener(this); + image_ids = { + {"global id 1", "remote id 1a"}, + {"global id 3", "remote id 3"}}; + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + mock_pool_watcher.init(nullptr); + + { + std::unique_lock locker{m_lock}; + m_cond.wait(locker, [&] { return refresh_sent; }); + } + + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3"); + + mock_refresh_images_request.on_finish->complete(0); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, Notify) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1"}, + {"global id 2", "remote id 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, image_ids, 0); + + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillOnce(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + Context *notify_ctx = nullptr; + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillOnce(Invoke([this, ¬ify_ctx](Context *ctx, int r) { + std::lock_guard locker{m_lock}; + ASSERT_EQ(nullptr, notify_ctx); + notify_ctx = ctx; + m_cond.notify_all(); + })); + expect_listener_handle_update( + mock_listener, "remote uuid", + {{"global id 1", "remote id 1a"}, {"global id 3", "remote id 3"}}, + {{"global id 1", "remote id 1"}, {"global id 2", "remote id 2"}}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + C_SaferCond flush_ctx; + m_threads->work_queue->queue(&flush_ctx, 0); + ASSERT_EQ(0, flush_ctx.wait()); + + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_DISABLED, "remote id 2", "global id 2"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3"); + notify_ctx->complete(0); + + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RegisterWatcherBlocklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, -EBLOCKLISTED); + + MockListener mock_listener(this); + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-EBLOCKLISTED, ctx.wait()); + ASSERT_TRUE(mock_pool_watcher.is_blocklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RegisterWatcherMissing) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, -ENOENT); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RegisterWatcherError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, -EINVAL); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RefreshBlocklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, -EBLOCKLISTED); + + MockListener mock_listener(this); + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-EBLOCKLISTED, ctx.wait()); + ASSERT_TRUE(mock_pool_watcher.is_blocklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RefreshMissing) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, -ENOENT); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RefreshError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, -EINVAL); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, Rewatch) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + expect_timer_add_event(mock_threads); + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0); + expect_listener_handle_update(mock_listener, "remote uuid", + {{"global id", "image id"}}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + MirroringWatcher::get_instance().handle_rewatch_complete(0); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RewatchBlocklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + MirroringWatcher::get_instance().handle_rewatch_complete(-EBLOCKLISTED); + ASSERT_TRUE(mock_pool_watcher.is_blocklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RewatchError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + expect_timer_add_event(mock_threads); + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0); + expect_listener_handle_update(mock_listener, "remote uuid", + {{"global id", "image id"}}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + MirroringWatcher::get_instance().handle_rewatch_complete(-EINVAL); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, DeferredRefresh) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + + EXPECT_CALL(mock_refresh_images_request, send()) + .WillOnce(Invoke([&mock_refresh_images_request]() { + *mock_refresh_images_request.image_ids = {}; + MirroringWatcher::get_instance().handle_rewatch_complete(0); + mock_refresh_images_request.on_finish->complete(0); + })); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + "remote uuid", mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_Throttler.cc b/src/test/rbd_mirror/test_mock_Throttler.cc new file mode 100644 index 000000000..ab562c18d --- /dev/null +++ b/src/test/rbd_mirror/test_mock_Throttler.cc @@ -0,0 +1,253 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2016 SUSE LINUX GmbH + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/Throttler.cc" + +namespace rbd { +namespace mirror { + +class TestMockThrottler : public TestMockFixture { +public: + typedef Throttler<librbd::MockTestImageCtx> MockThrottler; + +}; + +TEST_F(TestMockThrottler, Single_Sync) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + C_SaferCond on_start; + throttler.start_op("ns", "id", &on_start); + ASSERT_EQ(0, on_start.wait()); + throttler.finish_op("ns", "id"); +} + +TEST_F(TestMockThrottler, Multiple_Syncs) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(2); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("ns", "id2", &on_start2); + C_SaferCond on_start3; + throttler.start_op("ns", "id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("ns", "id4", &on_start4); + + ASSERT_EQ(0, on_start2.wait()); + throttler.finish_op("ns", "id2"); + ASSERT_EQ(0, on_start3.wait()); + throttler.finish_op("ns", "id3"); + ASSERT_EQ(0, on_start1.wait()); + throttler.finish_op("ns", "id1"); + ASSERT_EQ(0, on_start4.wait()); + throttler.finish_op("ns", "id4"); +} + +TEST_F(TestMockThrottler, Cancel_Running_Sync) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + C_SaferCond on_start; + throttler.start_op("ns", "id", &on_start); + ASSERT_EQ(0, on_start.wait()); + ASSERT_FALSE(throttler.cancel_op("ns", "id")); + throttler.finish_op("ns", "id"); +} + +TEST_F(TestMockThrottler, Cancel_Waiting_Sync) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(1); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("ns", "id2", &on_start2); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_TRUE(throttler.cancel_op("ns", "id2")); + ASSERT_EQ(-ECANCELED, on_start2.wait()); + throttler.finish_op("ns", "id1"); +} + +TEST_F(TestMockThrottler, Cancel_Running_Sync_Start_Waiting) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(1); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("ns", "id2", &on_start2); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_FALSE(throttler.cancel_op("ns", "id1")); + throttler.finish_op("ns", "id1"); + ASSERT_EQ(0, on_start2.wait()); + throttler.finish_op("ns", "id2"); +} + +TEST_F(TestMockThrottler, Duplicate) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(1); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + ASSERT_EQ(0, on_start1.wait()); + + C_SaferCond on_start2; + throttler.start_op("ns", "id1", &on_start2); + ASSERT_EQ(0, on_start2.wait()); + + C_SaferCond on_start3; + throttler.start_op("ns", "id2", &on_start3); + C_SaferCond on_start4; + throttler.start_op("ns", "id2", &on_start4); + ASSERT_EQ(-ENOENT, on_start3.wait()); + + throttler.finish_op("ns", "id1"); + ASSERT_EQ(0, on_start4.wait()); + throttler.finish_op("ns", "id2"); +} + +TEST_F(TestMockThrottler, Duplicate2) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(2); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + ASSERT_EQ(0, on_start1.wait()); + C_SaferCond on_start2; + throttler.start_op("ns", "id2", &on_start2); + ASSERT_EQ(0, on_start2.wait()); + + C_SaferCond on_start3; + throttler.start_op("ns", "id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("ns", "id3", &on_start4); // dup + ASSERT_EQ(-ENOENT, on_start3.wait()); + + C_SaferCond on_start5; + throttler.start_op("ns", "id4", &on_start5); + + throttler.finish_op("ns", "id1"); + ASSERT_EQ(0, on_start4.wait()); + + throttler.finish_op("ns", "id2"); + ASSERT_EQ(0, on_start5.wait()); + + C_SaferCond on_start6; + throttler.start_op("ns", "id5", &on_start6); + + throttler.finish_op("ns", "id3"); + ASSERT_EQ(0, on_start6.wait()); + + throttler.finish_op("ns", "id4"); + throttler.finish_op("ns", "id5"); +} + +TEST_F(TestMockThrottler, Increase_Max_Concurrent_Syncs) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(2); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("ns", "id2", &on_start2); + C_SaferCond on_start3; + throttler.start_op("ns", "id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("ns", "id4", &on_start4); + C_SaferCond on_start5; + throttler.start_op("ns", "id5", &on_start5); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_EQ(0, on_start2.wait()); + + throttler.set_max_concurrent_ops(4); + + ASSERT_EQ(0, on_start3.wait()); + ASSERT_EQ(0, on_start4.wait()); + + throttler.finish_op("ns", "id4"); + ASSERT_EQ(0, on_start5.wait()); + + throttler.finish_op("ns", "id1"); + throttler.finish_op("ns", "id2"); + throttler.finish_op("ns", "id3"); + throttler.finish_op("ns", "id5"); +} + +TEST_F(TestMockThrottler, Decrease_Max_Concurrent_Syncs) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(4); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("ns", "id2", &on_start2); + C_SaferCond on_start3; + throttler.start_op("ns", "id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("ns", "id4", &on_start4); + C_SaferCond on_start5; + throttler.start_op("ns", "id5", &on_start5); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_EQ(0, on_start2.wait()); + ASSERT_EQ(0, on_start3.wait()); + ASSERT_EQ(0, on_start4.wait()); + + throttler.set_max_concurrent_ops(2); + + throttler.finish_op("ns", "id1"); + throttler.finish_op("ns", "id2"); + throttler.finish_op("ns", "id3"); + + ASSERT_EQ(0, on_start5.wait()); + + throttler.finish_op("ns", "id4"); + throttler.finish_op("ns", "id5"); +} + +TEST_F(TestMockThrottler, Drain) { + MockThrottler throttler(g_ceph_context, "rbd_mirror_concurrent_image_syncs"); + throttler.set_max_concurrent_ops(1); + + C_SaferCond on_start1; + throttler.start_op("ns", "id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("ns", "id2", &on_start2); + + ASSERT_EQ(0, on_start1.wait()); + throttler.drain("ns", -ESTALE); + ASSERT_EQ(-ESTALE, on_start2.wait()); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_fixture.cc b/src/test/rbd_mirror/test_mock_fixture.cc new file mode 100644 index 000000000..9e308a63b --- /dev/null +++ b/src/test/rbd_mirror/test_mock_fixture.cc @@ -0,0 +1,64 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "test/librados_test_stub/LibradosTestStub.h" +#include "test/librados_test_stub/MockTestMemCluster.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::WithArg; + +TestMockFixture::TestClusterRef TestMockFixture::s_test_cluster; + +void TestMockFixture::SetUpTestCase() { + s_test_cluster = librados_test_stub::get_cluster(); + + // use a mock version of the in-memory rados client + librados_test_stub::set_cluster(boost::shared_ptr<librados::TestCluster>( + new ::testing::NiceMock<librados::MockTestMemCluster>())); + TestFixture::SetUpTestCase(); +} + +void TestMockFixture::TearDownTestCase() { + TestFixture::TearDownTestCase(); + librados_test_stub::set_cluster(s_test_cluster); +} + +void TestMockFixture::TearDown() { + // Mock rados client lives across tests -- reset it to initial state + librados::MockTestMemRadosClient *mock_rados_client = + get_mock_io_ctx(m_local_io_ctx).get_mock_rados_client(); + ASSERT_TRUE(mock_rados_client != nullptr); + + ::testing::Mock::VerifyAndClear(mock_rados_client); + mock_rados_client->default_to_dispatch(); + dynamic_cast<librados::MockTestMemCluster*>( + librados_test_stub::get_cluster().get())->default_to_dispatch(); + + TestFixture::TearDown(); +} + +void TestMockFixture::expect_test_features(librbd::MockImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, test_features(_, _)) + .WillRepeatedly(WithArg<0>(Invoke([&mock_image_ctx](uint64_t features) { + return (mock_image_ctx.features & features) != 0; + }))); +} + +librados::MockTestMemCluster& TestMockFixture::get_mock_cluster() { + librados::MockTestMemCluster* mock_cluster = dynamic_cast< + librados::MockTestMemCluster*>(librados_test_stub::get_cluster().get()); + ceph_assert(mock_cluster != nullptr); + return *mock_cluster; +} + +} // namespace mirror +} // namespace rbd + diff --git a/src/test/rbd_mirror/test_mock_fixture.h b/src/test/rbd_mirror/test_mock_fixture.h new file mode 100644 index 000000000..16b6dc6b8 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_fixture.h @@ -0,0 +1,72 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_TEST_RBD_MIRROR_TEST_MOCK_FIXTURE_H +#define CEPH_TEST_RBD_MIRROR_TEST_MOCK_FIXTURE_H + +#include "test/rbd_mirror/test_fixture.h" +#include "test/librados_test_stub/LibradosTestStub.h" +#include "common/WorkQueue.h" +#include "librbd/asio/ContextWQ.h" +#include <boost/shared_ptr.hpp> +#include <gmock/gmock.h> +#include "include/ceph_assert.h" + +namespace librados { +class TestRadosClient; +class MockTestMemCluster; +class MockTestMemIoCtxImpl; +class MockTestMemRadosClient; +} + +namespace librbd { +class MockImageCtx; +} + +ACTION_P(CopyInBufferlist, str) { + arg0->append(str); +} + +ACTION_P(CompleteContext, r) { + arg0->complete(r); +} + +ACTION_P2(CompleteContext, wq, r) { + auto context_wq = reinterpret_cast<librbd::asio::ContextWQ *>(wq); + context_wq->queue(arg0, r); +} + +ACTION_P(GetReference, ref_object) { + ref_object->get(); +} + +MATCHER_P(ContentsEqual, bl, "") { + // TODO fix const-correctness of bufferlist + return const_cast<bufferlist &>(arg).contents_equal( + const_cast<bufferlist &>(bl)); +} + +namespace rbd { +namespace mirror { + +class TestMockFixture : public TestFixture { +public: + typedef boost::shared_ptr<librados::TestCluster> TestClusterRef; + + static void SetUpTestCase(); + static void TearDownTestCase(); + + void TearDown() override; + + void expect_test_features(librbd::MockImageCtx &mock_image_ctx); + + librados::MockTestMemCluster& get_mock_cluster(); + +private: + static TestClusterRef s_test_cluster; +}; + +} // namespace mirror +} // namespace rbd + +#endif // CEPH_TEST_RBD_MIRROR_TEST_MOCK_FIXTURE_H |