diff options
Diffstat (limited to 'src/test/rbd_mirror')
40 files changed, 16852 insertions, 0 deletions
diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt new file mode 100644 index 00000000..1d8a9757 --- /dev/null +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -0,0 +1,97 @@ +set(rbd_mirror_test_srcs + test_ClusterWatcher.cc + test_PoolWatcher.cc + test_ImageDeleter.cc + test_ImageReplayer.cc + test_ImageSync.cc + test_InstanceWatcher.cc + test_Instances.cc + test_LeaderWatcher.cc + test_fixture.cc + image_map/test_Policy.cc + ) +add_library(rbd_mirror_test STATIC ${rbd_mirror_test_srcs}) +target_link_libraries(rbd_mirror_test + rbd_test_support + GTest::GTest) + +add_executable(unittest_rbd_mirror + test_main.cc + test_mock_fixture.cc + test_mock_ImageMap.cc + test_mock_ImageReplayer.cc + test_mock_ImageSync.cc + test_mock_ImageSyncThrottler.cc + test_mock_InstanceReplayer.cc + test_mock_InstanceWatcher.cc + test_mock_LeaderWatcher.cc + test_mock_PoolReplayer.cc + test_mock_PoolWatcher.cc + image_deleter/test_mock_SnapshotPurgeRequest.cc + image_deleter/test_mock_TrashMoveRequest.cc + image_deleter/test_mock_TrashRemoveRequest.cc + image_deleter/test_mock_TrashWatcher.cc + image_replayer/test_mock_BootstrapRequest.cc + image_replayer/test_mock_CreateImageRequest.cc + image_replayer/test_mock_EventPreprocessor.cc + image_replayer/test_mock_GetMirrorImageIdRequest.cc + image_replayer/test_mock_PrepareLocalImageRequest.cc + image_replayer/test_mock_PrepareRemoteImageRequest.cc + image_sync/test_mock_SyncPointCreateRequest.cc + image_sync/test_mock_SyncPointPruneRequest.cc + pool_watcher/test_mock_RefreshImagesRequest.cc + ) +add_ceph_unittest(unittest_rbd_mirror) + +add_dependencies(unittest_rbd_mirror + cls_journal + cls_lock + cls_rbd) +target_link_libraries(unittest_rbd_mirror + rbd_mirror_test + rados_test_stub + rbd_mirror_internal + rbd_mirror_types + rbd_api + rbd_internal + rbd_test_mock + journal + journal_test_mock + cls_rbd_client + cls_lock_client + cls_journal_client + rbd_types + librados + osdc + global + radostest-cxx + ) + +add_executable(ceph_test_rbd_mirror + test_main.cc + ) + +target_link_libraries(ceph_test_rbd_mirror + rbd_mirror_test + rbd_mirror_internal + rbd_mirror_types + rbd_api + rbd_internal + journal + cls_rbd_client + cls_journal_client + rbd_types + librados + radostest-cxx + ${UNITTEST_LIBS} + ) + +add_executable(ceph_test_rbd_mirror_random_write + random_write.cc) +target_link_libraries(ceph_test_rbd_mirror_random_write + librbd librados global) + +install(TARGETS + ceph_test_rbd_mirror + ceph_test_rbd_mirror_random_write + DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/test/rbd_mirror/image_deleter/test_mock_SnapshotPurgeRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_SnapshotPurgeRequest.cc new file mode 100644 index 00000000..35850d3a --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_SnapshotPurgeRequest.cc @@ -0,0 +1,397 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockExclusiveLock.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockImageState.h" +#include "test/librbd/mock/MockOperations.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + static MockTestImageCtx *s_instance; + static MockTestImageCtx *create(const std::string &image_name, + const std::string &image_id, + const char *snap, librados::IoCtx& p, + bool read_only) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + s_instance = this; + } +}; + +MockTestImageCtx *MockTestImageCtx::s_instance = nullptr; + +} // anonymous namespace +} // namespace librbd + +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::InSequence; +using ::testing::WithArg; + +class TestMockImageDeleterSnapshotPurgeRequest : public TestMockFixture { +public: + typedef SnapshotPurgeRequest<librbd::MockTestImageCtx> MockSnapshotPurgeRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_set_journal_policy(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, set_journal_policy(_)) + .WillOnce(Invoke([](librbd::journal::Policy* policy) { + delete policy; + })); + } + + void expect_open(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, open(true, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_close(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, close(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_acquire_lock(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, acquire_lock(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_get_snap_namespace(librbd::MockTestImageCtx &mock_image_ctx, + uint64_t snap_id, + const cls::rbd::SnapshotNamespace &snap_namespace, + int r) { + EXPECT_CALL(mock_image_ctx, get_snap_namespace(snap_id, _)) + .WillOnce(WithArg<1>(Invoke([snap_namespace, r](cls::rbd::SnapshotNamespace *ns) { + *ns = snap_namespace; + return r; + }))); + } + + void expect_get_snap_name(librbd::MockTestImageCtx &mock_image_ctx, + uint64_t snap_id, const std::string& name, + int r) { + EXPECT_CALL(mock_image_ctx, get_snap_name(snap_id, _)) + .WillOnce(WithArg<1>(Invoke([name, r](std::string *n) { + *n = name; + return r; + }))); + } + + void expect_is_snap_protected(librbd::MockTestImageCtx &mock_image_ctx, + uint64_t snap_id, bool is_protected, int r) { + EXPECT_CALL(mock_image_ctx, is_snap_protected(snap_id, _)) + .WillOnce(WithArg<1>(Invoke([is_protected, r](bool *prot) { + *prot = is_protected; + return r; + }))); + } + + void expect_snap_unprotect(librbd::MockTestImageCtx &mock_image_ctx, + const cls::rbd::SnapshotNamespace& ns, + const std::string& name, int r) { + EXPECT_CALL(*mock_image_ctx.operations, execute_snap_unprotect(ns, name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_snap_remove(librbd::MockTestImageCtx &mock_image_ctx, + const cls::rbd::SnapshotNamespace& ns, + const std::string& name, int r) { + EXPECT_CALL(*mock_image_ctx.operations, execute_snap_remove(ns, name, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_start_op(librbd::MockTestImageCtx &mock_image_ctx, bool success) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, start_op(_)) + .WillOnce(Invoke([success](int* r) { + if (!success) { + *r = -EROFS; + return static_cast<FunctionContext*>(nullptr); + } + return new FunctionContext([](int r) {}); + })); + } + + void expect_destroy(librbd::MockTestImageCtx& mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, destroy()); + } + + librbd::ImageCtx *m_local_image_ctx; +}; + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, Success) { + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap2", 2, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 2, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 2, "snap2", 0); + expect_is_snap_protected(mock_image_ctx, 2, false, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap2", + 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, true, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_unprotect(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, + "snap1", 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap1", + 0); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, OpenError) { + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, -EPERM); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, AcquireLockError) { + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, -EPERM); + expect_close(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SnapUnprotectBusy) { + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, true, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_unprotect(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, + "snap1", -EBUSY); + + expect_close(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EBUSY, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SnapUnprotectError) { + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_PROTECTED, 0, {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, true, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_unprotect(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, + "snap1", -EPERM); + + expect_close(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, SnapRemoveError) { + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, false, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap1", + -EINVAL); + + expect_close(mock_image_ctx, -EPERM); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterSnapshotPurgeRequest, CloseError) { + { + RWLock::WLocker snap_locker(m_local_image_ctx->snap_lock); + m_local_image_ctx->add_snap(cls::rbd::UserSnapshotNamespace{}, "snap1", 1, + 0, {}, RBD_PROTECTION_STATUS_UNPROTECTED, 0, + {}); + } + + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_acquire_lock(mock_image_ctx, 0); + + expect_get_snap_namespace(mock_image_ctx, 1, + cls::rbd::UserSnapshotNamespace{}, 0); + expect_get_snap_name(mock_image_ctx, 1, "snap1", 0); + expect_is_snap_protected(mock_image_ctx, 1, false, 0); + expect_start_op(mock_image_ctx, true); + expect_snap_remove(mock_image_ctx, cls::rbd::UserSnapshotNamespace{}, "snap1", + 0); + + expect_close(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockSnapshotPurgeRequest::create(m_local_io_ctx, mock_image_ctx.id, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc new file mode 100644 index 00000000..b2ec9692 --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc @@ -0,0 +1,738 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/TrashWatcher.h" +#include "librbd/journal/ResetRequest.h" +#include "librbd/trash/MoveRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockExclusiveLock.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockImageState.h" +#include "test/librbd/mock/MockOperations.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + static MockTestImageCtx *s_instance; + static MockTestImageCtx *create(const std::string &image_name, + const std::string &image_id, + const char *snap, librados::IoCtx& p, + bool read_only) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + s_instance = this; + } +}; + +MockTestImageCtx *MockTestImageCtx::s_instance = nullptr; + +} // anonymous namespace + +template <> +struct Journal<librbd::MockTestImageCtx> { + static Journal *s_instance; + + static void get_tag_owner(librados::IoCtx &io_ctx, + const std::string &image_id, + std::string *mirror_uuid, + ContextWQ *work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->get_tag_owner(image_id, mirror_uuid, on_finish); + } + + MOCK_METHOD3(get_tag_owner, void(const std::string &, std::string *, + Context *)); + + Journal() { + s_instance = this; + } +}; + +Journal<librbd::MockTestImageCtx>* Journal<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct TrashWatcher<MockTestImageCtx> { + static TrashWatcher* s_instance; + static void notify_image_added(librados::IoCtx&, const std::string& image_id, + const cls::rbd::TrashImageSpec& spec, + Context *ctx) { + ceph_assert(s_instance != nullptr); + s_instance->notify_image_added(image_id, spec, ctx); + } + + MOCK_METHOD3(notify_image_added, void(const std::string&, + const cls::rbd::TrashImageSpec&, + Context*)); + + TrashWatcher() { + s_instance = this; + } +}; + +TrashWatcher<MockTestImageCtx>* TrashWatcher<MockTestImageCtx>::s_instance = nullptr; + +namespace journal { + +template <> +struct ResetRequest<MockTestImageCtx> { + static ResetRequest* s_instance; + Context* on_finish = nullptr; + + static ResetRequest* create(librados::IoCtx &io_ctx, + const std::string &image_id, + const std::string &client_id, + const std::string &mirror_uuid, + ContextWQ *op_work_queue, Context *on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_EQ(librbd::Journal<>::LOCAL_MIRROR_UUID, mirror_uuid); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + ResetRequest() { + s_instance = this; + } +}; + +ResetRequest<MockTestImageCtx>* ResetRequest<MockTestImageCtx>::s_instance = nullptr; + +} // namespace journal + +namespace trash { + +template <> +struct MoveRequest<MockTestImageCtx> { + static MoveRequest* s_instance; + Context* on_finish = nullptr; + + typedef boost::optional<utime_t> DefermentEndTime; + + static MoveRequest* create(librados::IoCtx& io_ctx, + const std::string& image_id, + const cls::rbd::TrashImageSpec& trash_image_spec, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->construct(image_id, trash_image_spec); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD2(construct, void(const std::string&, + const cls::rbd::TrashImageSpec&)); + MOCK_METHOD0(send, void()); + + MoveRequest() { + s_instance = this; + } +}; + +MoveRequest<MockTestImageCtx>* MoveRequest<MockTestImageCtx>::s_instance = nullptr; + +} // namespace trash +} // namespace librbd + +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageDeleterTrashMoveRequest : public TestMockFixture { +public: + typedef TrashMoveRequest<librbd::MockTestImageCtx> MockTrashMoveRequest; + typedef librbd::Journal<librbd::MockTestImageCtx> MockJournal; + typedef librbd::journal::ResetRequest<librbd::MockTestImageCtx> MockJournalResetRequest; + typedef librbd::trash::MoveRequest<librbd::MockTestImageCtx> MockLibrbdTrashMoveRequest; + typedef librbd::TrashWatcher<librbd::MockTestImageCtx> MockTrashWatcher; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_mirror_image_get_image_id(const std::string& image_id, int r) { + bufferlist bl; + encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get_image_id"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_get_tag_owner(MockJournal &mock_journal, + const std::string &image_id, + const std::string &tag_owner, int r) { + EXPECT_CALL(mock_journal, get_tag_owner(image_id, _, _)) + .WillOnce(WithArgs<1, 2>(Invoke([this, tag_owner, r](std::string *owner, Context *on_finish) { + *owner = tag_owner; + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_set_journal_policy(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, set_journal_policy(_)) + .WillOnce(Invoke([](librbd::journal::Policy* policy) { + delete policy; + })); + } + + void expect_open(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, open(true, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_close(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, close(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_block_requests(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, block_requests(0)).Times(1); + } + + void expect_acquire_lock(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, acquire_lock(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_destroy(librbd::MockTestImageCtx& mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, destroy()); + } + + void expect_mirror_image_set(const std::string& image_id, + const std::string& global_image_id, + cls::rbd::MirrorImageState mirror_image_state, + int r) { + cls::rbd::MirrorImage mirror_image; + mirror_image.global_image_id = global_image_id; + mirror_image.state = mirror_image_state; + + bufferlist bl; + encode(image_id, bl); + encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_set"), ContentsEqual(bl), _, _)) + .WillOnce(Return(r)); + } + + void expect_mirror_image_remove(librados::IoCtx &ioctx, int r) { + EXPECT_CALL(get_mock_io_ctx(ioctx), + exec(StrEq("rbd_mirroring"), _, StrEq("rbd"), StrEq("mirror_image_remove"), + _, _, _)) + .WillOnce(Return(r)); + } + + void expect_journal_reset(MockJournalResetRequest& mock_journal_reset_request, + int r) { + EXPECT_CALL(mock_journal_reset_request, send()) + .WillOnce(Invoke([this, &mock_journal_reset_request, r]() { + m_threads->work_queue->queue(mock_journal_reset_request.on_finish, r); + })); + } + + void expect_trash_move(MockLibrbdTrashMoveRequest& mock_trash_move_request, + const std::string& image_name, + const std::string& image_id, + const boost::optional<uint32_t>& delay, int r) { + EXPECT_CALL(mock_trash_move_request, construct(image_id, _)) + .WillOnce(WithArg<1>(Invoke([image_name, delay](const cls::rbd::TrashImageSpec& spec) { + ASSERT_EQ(cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, + spec.source); + ASSERT_EQ(image_name, spec.name); + if (delay) { + utime_t time{spec.deletion_time}; + time += *delay; + ASSERT_TRUE(time == spec.deferment_end_time); + } else { + ASSERT_EQ(spec.deletion_time, spec.deferment_end_time); + } + }))); + EXPECT_CALL(mock_trash_move_request, send()) + .WillOnce(Invoke([this, &mock_trash_move_request, r]() { + m_threads->work_queue->queue(mock_trash_move_request.on_finish, r); + })); + } + + void expect_notify_image_added(MockTrashWatcher& mock_trash_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_trash_watcher, notify_image_added(image_id, _, _)) + .WillOnce(WithArg<2>(Invoke([this](Context *ctx) { + m_threads->work_queue->queue(ctx, 0); + }))); + } + + librbd::ImageCtx *m_local_image_ctx; +}; + +TEST_F(TestMockImageDeleterTrashMoveRequest, Success) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + expect_mirror_image_remove(m_local_io_ctx, 0); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetImageIdDNE) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -ENOENT); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetImageIdError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerLocalPrimary) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::LOCAL_MIRROR_UUID, 0); + + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerOrphan) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + false, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerDNE) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", -ENOENT); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + expect_mirror_image_remove(m_local_io_ctx, 0); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, DisableMirrorImageError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, ResetJournalError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, OpenImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, AcquireLockError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, -EINVAL); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, TrashMoveError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, -EINVAL); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, RemoveMirrorImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + expect_mirror_image_remove(m_local_io_ctx, -EINVAL); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, CloseImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + {}, 0); + expect_mirror_image_remove(m_local_io_ctx, 0); + + expect_close(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, DelayedDelation) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.config.set_val("rbd_mirroring_delete_delay", "600"); + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", + 600, 0); + + expect_mirror_image_remove(m_local_io_ctx, 0); + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashRemoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashRemoveRequest.cc new file mode 100644 index 00000000..be875704 --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashRemoveRequest.cc @@ -0,0 +1,452 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/TrashWatcher.h" +#include "librbd/Utils.h" +#include "librbd/trash/RemoveRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h" +#include "tools/rbd_mirror/image_deleter/TrashRemoveRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +template<> +struct TrashWatcher<MockTestImageCtx> { + static TrashWatcher* s_instance; + static void notify_image_removed(librados::IoCtx&, + const std::string& image_id, Context *ctx) { + ceph_assert(s_instance != nullptr); + s_instance->notify_image_removed(image_id, ctx); + } + + MOCK_METHOD2(notify_image_removed, void(const std::string&, Context*)); + + TrashWatcher() { + s_instance = this; + } +}; + +TrashWatcher<MockTestImageCtx>* TrashWatcher<MockTestImageCtx>::s_instance = nullptr; + +namespace trash { + +template <> +struct RemoveRequest<librbd::MockTestImageCtx> { + static RemoveRequest *s_instance; + Context *on_finish = nullptr; + + static RemoveRequest *create(librados::IoCtx &io_ctx, + const std::string &image_id, + ContextWQ *work_queue, + bool force, + librbd::ProgressContext &progress_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_TRUE(force); + s_instance->construct(image_id); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); + + RemoveRequest() { + s_instance = this; + } +}; + +RemoveRequest<librbd::MockTestImageCtx>* RemoveRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace trash +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_deleter { + +template <> +struct SnapshotPurgeRequest<librbd::MockTestImageCtx> { + static SnapshotPurgeRequest *s_instance; + Context *on_finish = nullptr; + + static SnapshotPurgeRequest *create(librados::IoCtx &io_ctx, + const std::string &image_id, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->construct(image_id); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); + + SnapshotPurgeRequest() { + s_instance = this; + } +}; + +SnapshotPurgeRequest<librbd::MockTestImageCtx>* SnapshotPurgeRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_deleter/TrashRemoveRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::InSequence; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageDeleterTrashRemoveRequest : public TestMockFixture { +public: + typedef TrashRemoveRequest<librbd::MockTestImageCtx> MockTrashRemoveRequest; + typedef SnapshotPurgeRequest<librbd::MockTestImageCtx> MockSnapshotPurgeRequest; + typedef librbd::TrashWatcher<librbd::MockTestImageCtx> MockTrashWatcher; + typedef librbd::trash::RemoveRequest<librbd::MockTestImageCtx> MockLibrbdTrashRemoveRequest; + + void expect_trash_get(const cls::rbd::TrashImageSpec& trash_spec, int r) { + using ceph::encode; + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(StrEq(RBD_TRASH), _, StrEq("rbd"), + StrEq("trash_get"), _, _, _)) + .WillOnce(WithArg<5>(Invoke([trash_spec, r](bufferlist* bl) { + encode(trash_spec, *bl); + return r; + }))); + } + + void expect_trash_state_set(const std::string& image_id, int r) { + bufferlist in_bl; + encode(image_id, in_bl); + encode(cls::rbd::TRASH_IMAGE_STATE_REMOVING, in_bl); + encode(cls::rbd::TRASH_IMAGE_STATE_NORMAL, in_bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(StrEq(RBD_TRASH), _, StrEq("rbd"), + StrEq("trash_state_set"), + ContentsEqual(in_bl), _, _)) + .WillOnce(Return(r)); + } + + void expect_get_snapcontext(const std::string& image_id, + const ::SnapContext &snapc, int r) { + bufferlist bl; + encode(snapc, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(librbd::util::header_name(image_id), _, StrEq("rbd"), + StrEq("get_snapcontext"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_snapshot_purge(MockSnapshotPurgeRequest &snapshot_purge_request, + const std::string &image_id, int r) { + EXPECT_CALL(snapshot_purge_request, construct(image_id)); + EXPECT_CALL(snapshot_purge_request, send()) + .WillOnce(Invoke([this, &snapshot_purge_request, r]() { + m_threads->work_queue->queue( + snapshot_purge_request.on_finish, r); + })); + } + + void expect_image_remove(MockLibrbdTrashRemoveRequest &image_remove_request, + const std::string &image_id, int r) { + EXPECT_CALL(image_remove_request, construct(image_id)); + EXPECT_CALL(image_remove_request, send()) + .WillOnce(Invoke([this, &image_remove_request, r]() { + m_threads->work_queue->queue( + image_remove_request.on_finish, r); + })); + } + + void expect_notify_image_removed(MockTrashWatcher& mock_trash_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_trash_watcher, notify_image_removed(image_id, _)) + .WillOnce(WithArg<1>(Invoke([this](Context *ctx) { + m_threads->work_queue->queue(ctx, 0); + }))); + } + +}; + +TEST_F(TestMockImageDeleterTrashRemoveRequest, Success) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", 0); + + MockLibrbdTrashRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_removed(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashDNE) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, -ENOENT); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, -EPERM); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashSourceIncorrect) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_USER, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashStateIncorrect) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + trash_image_spec.state = cls::rbd::TRASH_IMAGE_STATE_RESTORING; + expect_trash_get(trash_image_spec, 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EBUSY, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_RETRY_IMMEDIATELY, error_result); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashSetStateDNE) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", -ENOENT); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, TrashSetStateError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", -EPERM); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, GetSnapContextDNE) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, -ENOENT); + + MockLibrbdTrashRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_removed(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, GetSnapContextError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, PurgeSnapshotBusy) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", -EBUSY); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EBUSY, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_RETRY_IMMEDIATELY, error_result); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, PurgeSnapshotError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashRemoveRequest, RemoveError) { + InSequence seq; + + cls::rbd::TrashImageSpec trash_image_spec{ + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "image name", {}, {}}; + expect_trash_get(trash_image_spec, 0); + + expect_trash_state_set("image id", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", 0); + + MockLibrbdTrashRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockTrashRemoveRequest::create(m_local_io_ctx, "image id", + &error_result, + m_threads->work_queue, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc new file mode 100644 index 00000000..034c3e54 --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc @@ -0,0 +1,518 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "librbd/TrashWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/TrashWatcher.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +struct MockTrashWatcher { + static MockTrashWatcher *s_instance; + static MockTrashWatcher &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + MockTrashWatcher() { + s_instance = this; + } + + MOCK_CONST_METHOD0(is_unregistered, bool()); + MOCK_METHOD1(register_watch, void(Context*)); + MOCK_METHOD1(unregister_watch, void(Context*)); +}; + +template <> +struct TrashWatcher<MockTestImageCtx> { + static TrashWatcher *s_instance; + + TrashWatcher(librados::IoCtx &io_ctx, ::MockContextWQ *work_queue) { + s_instance = this; + } + virtual ~TrashWatcher() { + } + + static TrashWatcher<MockTestImageCtx> &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + virtual void handle_rewatch_complete(int r) = 0; + + virtual void handle_image_added(const std::string &image_id, + const cls::rbd::TrashImageSpec& spec) = 0; + virtual void handle_image_removed(const std::string &image_id) = 0; + + bool is_unregistered() const { + return MockTrashWatcher::get_instance().is_unregistered(); + } + void register_watch(Context *ctx) { + MockTrashWatcher::get_instance().register_watch(ctx); + } + void unregister_watch(Context *ctx) { + MockTrashWatcher::get_instance().unregister_watch(ctx); + } +}; + +MockTrashWatcher *MockTrashWatcher::s_instance = nullptr; +TrashWatcher<MockTestImageCtx> *TrashWatcher<MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + Mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_deleter/TrashWatcher.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageDeleterTrashWatcher : public TestMockFixture { +public: + typedef TrashWatcher<librbd::MockTestImageCtx> MockTrashWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef librbd::MockTrashWatcher MockLibrbdTrashWatcher; + typedef librbd::TrashWatcher<librbd::MockTestImageCtx> LibrbdTrashWatcher; + + struct MockListener : TrashListener { + MOCK_METHOD2(handle_trash_image, void(const std::string&, const utime_t&)); + }; + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_trash_watcher_is_unregistered(MockLibrbdTrashWatcher &mock_trash_watcher, + bool unregistered) { + EXPECT_CALL(mock_trash_watcher, is_unregistered()) + .WillOnce(Return(unregistered)); + } + + void expect_trash_watcher_register(MockLibrbdTrashWatcher &mock_trash_watcher, + int r) { + EXPECT_CALL(mock_trash_watcher, register_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_trash_watcher_unregister(MockLibrbdTrashWatcher &mock_trash_watcher, + int r) { + EXPECT_CALL(mock_trash_watcher, unregister_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_create_trash(librados::IoCtx &io_ctx, int r) { + EXPECT_CALL(get_mock_io_ctx(io_ctx), create(RBD_TRASH, false)) + .WillOnce(Return(r)); + } + + void expect_trash_list(librados::IoCtx &io_ctx, + const std::string& last_image_id, + std::map<std::string, cls::rbd::TrashImageSpec>&& images, + int r) { + bufferlist bl; + encode(last_image_id, bl); + encode(static_cast<size_t>(1024), bl); + + bufferlist out_bl; + encode(images, out_bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_TRASH, _, StrEq("rbd"), StrEq("trash_list"), + ContentsEqual(bl), _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + + void expect_timer_add_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = + new FunctionContext([this, ctx](int r) { + Mutex::Locker timer_locker(m_threads->timer_lock); + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), + ReturnArg<1>())); + } + + void expect_handle_trash_image(MockListener& mock_listener, + const std::string& global_image_id) { + EXPECT_CALL(mock_listener, handle_trash_image(global_image_id, _)); + } + + int when_shut_down(MockTrashWatcher &mock_trash_watcher) { + C_SaferCond ctx; + mock_trash_watcher.shut_down(&ctx); + return ctx.wait(); + } + +}; + +TEST_F(TestMockImageDeleterTrashWatcher, EmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, NonEmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + MockListener mock_listener; + expect_handle_trash_image(mock_listener, "image0"); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + std::map<std::string, cls::rbd::TrashImageSpec> images; + images["image0"] = {cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "name", {}, {}}; + for (auto idx = 1; idx < 1024; ++idx) { + images["image" + stringify(idx)] = {}; + } + expect_trash_list(m_local_io_ctx, "", std::move(images), 0); + + images.clear(); + for (auto idx = 1024; idx < 2000; ++idx) { + images["image" + stringify(idx)] = {}; + } + expect_trash_list(m_local_io_ctx, "image999", std::move(images), 0); + + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, Notify) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + MockListener mock_listener; + expect_handle_trash_image(mock_listener, "image1"); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + LibrbdTrashWatcher::get_instance().handle_image_added( + "image1", {cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "name", {}, {}}); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, CreateBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, -EBLACKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, CreateDNE) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, -ENOENT); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, CreateError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, -EINVAL); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RegisterWatcherBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, -EBLACKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RegisterWatcherError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, -EINVAL); + expect_timer_add_event(mock_threads); + + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, TrashListBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, -EBLACKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, TrashListError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, -EINVAL); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, false); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, Rewatch) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, false); + expect_trash_list(m_local_io_ctx, "", {}, 0); + LibrbdTrashWatcher::get_instance().handle_rewatch_complete(0); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RewatchBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + LibrbdTrashWatcher::get_instance().handle_rewatch_complete(-EBLACKLISTED); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_map/test_Policy.cc b/src/test/rbd_mirror/image_map/test_Policy.cc new file mode 100644 index 00000000..4535ab7a --- /dev/null +++ b/src/test/rbd_mirror/image_map/test_Policy.cc @@ -0,0 +1,377 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/Context.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/image_map/Types.h" +#include "tools/rbd_mirror/image_map/SimplePolicy.h" +#include "include/stringify.h" +#include "common/Thread.h" + +void register_test_image_policy() { +} + +namespace rbd { +namespace mirror { +namespace image_map { + +class TestImageMapPolicy : public TestFixture { +public: + void SetUp() override { + TestFixture::SetUp(); + + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", + "0")); + + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + std::string policy_type = cct->_conf.get_val<string>("rbd_mirror_image_policy_type"); + + if (policy_type == "none" || policy_type == "simple") { + m_policy = image_map::SimplePolicy::create(m_local_io_ctx); + } else { + ceph_abort(); + } + + m_policy->init({}); + } + + void TearDown() override { + TestFixture::TearDown(); + delete m_policy; + } + + void map_image(const std::string &global_image_id) { + ASSERT_TRUE(m_policy->add_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + } + + void unmap_image(const std::string &global_image_id) { + ASSERT_TRUE(m_policy->remove_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_REMOVE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + } + + void shuffle_image(const std::string &global_image_id) { + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + } + + Policy *m_policy; +}; + +TEST_F(TestImageMapPolicy, NegativeLookup) { + const std::string global_image_id = "global id 1"; + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id == UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, Init) { + const std::string global_image_id = "global id 1"; + + m_policy->init({{global_image_id, {"9876", {}, {}}}}); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +TEST_F(TestImageMapPolicy, MapImage) { + const std::string global_image_id = "global id 1"; + + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, UnmapImage) { + const std::string global_image_id = "global id 1"; + + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + + // unmap image + unmap_image(global_image_id); + + info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id == UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, ShuffleImageAddInstance) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", "global id 6" + }; + + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + + for (auto const &global_image_id : shuffle_global_image_ids) { + shuffle_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } +} + +TEST_F(TestImageMapPolicy, ShuffleImageRemoveInstance) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5" + }; + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + + for (auto const &global_image_id : shuffle_global_image_ids) { + shuffle_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + // record which of the images got migrated to the new instance + std::set<std::string> remapped_global_image_ids; + for (auto const &global_image_id: shuffle_global_image_ids) { + LookupInfo info = m_policy->lookup(global_image_id); + if (info.instance_id == "9876") { + remapped_global_image_ids.emplace(global_image_id); + } + } + + shuffle_global_image_ids.clear(); + m_policy->remove_instances({"9876"}, &shuffle_global_image_ids); + + ASSERT_TRUE(shuffle_global_image_ids == remapped_global_image_ids); + + for (auto const &global_image_id : shuffle_global_image_ids) { + shuffle_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } +} + +TEST_F(TestImageMapPolicy, RetryMapUpdate) { + const std::string global_image_id = "global id 1"; + + ASSERT_TRUE(m_policy->add_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + // on-disk map update failed + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EIO)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, MapFailureAndUnmap) { + const std::string global_image_id = "global id 1"; + + ASSERT_TRUE(m_policy->add_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + m_policy->remove_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EBLACKLISTED)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, -ENOENT)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_TRUE(m_policy->remove_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_REMOVE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +TEST_F(TestImageMapPolicy, ReshuffleWithMapFailure) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6" + }; + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_FALSE(shuffle_global_image_ids.empty()); + + const std::string global_image_id = *(shuffle_global_image_ids.begin()); + shuffle_global_image_ids.clear(); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + // peer unavailable + m_policy->remove_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EBLACKLISTED)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +TEST_F(TestImageMapPolicy, ShuffleFailureAndRemove) { + std::set<std::string> global_image_ids { + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6" + }; + + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({stringify(m_local_io_ctx.get_instance_id())}, + &shuffle_global_image_ids); + for (auto const &global_image_id : global_image_ids) { + // map image + map_image(global_image_id); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id != UNMAPPED_INSTANCE_ID); + } + + m_policy->add_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_FALSE(shuffle_global_image_ids.empty()); + + std::string global_image_id = *(shuffle_global_image_ids.begin()); + shuffle_global_image_ids.clear(); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + // peer unavailable + m_policy->remove_instances({"9876"}, &shuffle_global_image_ids); + ASSERT_TRUE(shuffle_global_image_ids.empty()); + + ASSERT_TRUE(m_policy->finish_action(global_image_id, -EBLACKLISTED)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_TRUE(m_policy->remove_image(global_image_id)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_REMOVE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); + + LookupInfo info = m_policy->lookup(global_image_id); + ASSERT_TRUE(info.instance_id == UNMAPPED_INSTANCE_ID); +} + +TEST_F(TestImageMapPolicy, InitialInstanceUpdate) { + const std::string global_image_id = "global id 1"; + + m_policy->init({{global_image_id, {"9876", {}, {}}}}); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + + auto instance_id = stringify(m_local_io_ctx.get_instance_id()); + std::set<std::string> shuffle_global_image_ids; + m_policy->add_instances({instance_id}, &shuffle_global_image_ids); + + ASSERT_EQ(0U, shuffle_global_image_ids.size()); + ASSERT_TRUE(m_policy->finish_action(global_image_id, -ENOENT)); + + ASSERT_EQ(ACTION_TYPE_RELEASE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_MAP_UPDATE, m_policy->start_action(global_image_id)); + ASSERT_TRUE(m_policy->finish_action(global_image_id, 0)); + + ASSERT_EQ(ACTION_TYPE_ACQUIRE, m_policy->start_action(global_image_id)); + ASSERT_FALSE(m_policy->finish_action(global_image_id, 0)); +} + +} // namespace image_map +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc new file mode 100644 index 00000000..6c7c30a2 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -0,0 +1,1195 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/BootstrapRequest.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/IsPrimaryRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenImageRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal + +namespace util { + +static std::string s_image_id; + +template <> +std::string generate_image_id<MockTestImageCtx>(librados::IoCtx&) { + ceph_assert(!s_image_id.empty()); + return s_image_id; +} + +} // namespace util +} // namespace librbd + +namespace rbd { +namespace mirror { + +class ProgressContext; + +template <> +struct Threads<librbd::MockTestImageCtx> { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +template<> +struct ImageSync<librbd::MockTestImageCtx> { + static ImageSync* s_instance; + Context *on_finish = nullptr; + + static ImageSync* create( + librbd::MockTestImageCtx *local_image_ctx, + librbd::MockTestImageCtx *remote_image_ctx, + SafeTimer *timer, Mutex *timer_lock, + const std::string &mirror_uuid, ::journal::MockJournaler *journaler, + librbd::journal::MirrorPeerClientMeta *client_meta, ContextWQ *work_queue, + InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher, + Context *on_finish, ProgressContext *progress_ctx) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + ImageSync() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~ImageSync() { + s_instance = nullptr; + } + + MOCK_METHOD0(get, void()); + MOCK_METHOD0(put, void()); + MOCK_METHOD0(send, void()); + MOCK_METHOD0(cancel, void()); +}; + +ImageSync<librbd::MockTestImageCtx>* + ImageSync<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { +}; + +namespace image_replayer { + +template<> +struct CloseImageRequest<librbd::MockTestImageCtx> { + static CloseImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + s_instance->construct(*image_ctx); + return s_instance; + } + + CloseImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~CloseImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD1(construct, void(librbd::MockTestImageCtx *image_ctx)); + MOCK_METHOD0(send, void()); +}; + +template<> +struct CreateImageRequest<librbd::MockTestImageCtx> { + static CreateImageRequest* s_instance; + Context *on_finish = nullptr; + + static CreateImageRequest* create(Threads<librbd::MockTestImageCtx>* threads, + librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + librbd::MockTestImageCtx *remote_image_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + s_instance->construct(local_image_id); + return s_instance; + } + + CreateImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~CreateImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); +}; + +template<> +struct IsPrimaryRequest<librbd::MockTestImageCtx> { + static IsPrimaryRequest* s_instance; + bool *primary = nullptr; + Context *on_finish = nullptr; + + static IsPrimaryRequest* create(librbd::MockTestImageCtx *image_ctx, + bool *primary, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->primary = primary; + s_instance->on_finish = on_finish; + return s_instance; + } + + IsPrimaryRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~IsPrimaryRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct OpenImageRequest<librbd::MockTestImageCtx> { + static OpenImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static OpenImageRequest* create(librados::IoCtx &io_ctx, + librbd::MockTestImageCtx **image_ctx, + const std::string &image_id, + bool read_only, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + s_instance->construct(io_ctx, image_id); + return s_instance; + } + + OpenImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~OpenImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); + MOCK_METHOD0(send, void()); +}; + +template<> +struct OpenLocalImageRequest<librbd::MockTestImageCtx> { + static OpenLocalImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static OpenLocalImageRequest* create(librados::IoCtx &local_io_ctx, + librbd::MockTestImageCtx **local_image_ctx, + const std::string &local_image_id, + ContextWQ *work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = local_image_ctx; + s_instance->on_finish = on_finish; + s_instance->construct(local_io_ctx, local_image_id); + return s_instance; + } + + OpenLocalImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~OpenLocalImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); + MOCK_METHOD0(send, void()); +}; + +CloseImageRequest<librbd::MockTestImageCtx>* + CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +CreateImageRequest<librbd::MockTestImageCtx>* + CreateImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +IsPrimaryRequest<librbd::MockTestImageCtx>* + IsPrimaryRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +OpenImageRequest<librbd::MockTestImageCtx>* + OpenImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +OpenLocalImageRequest<librbd::MockTestImageCtx>* + OpenLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/BootstrapRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::StrEq; +using ::testing::WithArg; + +MATCHER_P(IsSameIoCtx, io_ctx, "") { + return &get_mock_io_ctx(arg) == &get_mock_io_ctx(*io_ctx); +} + +class TestMockImageReplayerBootstrapRequest : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest; + typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest; + typedef CreateImageRequest<librbd::MockTestImageCtx> MockCreateImageRequest; + typedef ImageSync<librbd::MockTestImageCtx> MockImageSync; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef IsPrimaryRequest<librbd::MockTestImageCtx> MockIsPrimaryRequest; + typedef OpenImageRequest<librbd::MockTestImageCtx> MockOpenImageRequest; + typedef OpenLocalImageRequest<librbd::MockTestImageCtx> MockOpenLocalImageRequest; + typedef std::list<cls::journal::Tag> Tags; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void create_local_image() { + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_get_tags(::journal::MockJournaler &mock_journaler, + uint64_t tag_class, const Tags& tags, + int r) { + EXPECT_CALL(mock_journaler, get_tags(tag_class, _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([tags](Tags *out_tags) { + *out_tags = tags; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_register_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_journaler_unregister_client(::journal::MockJournaler &mock_journaler, + int r) { + EXPECT_CALL(mock_journaler, unregister_client(_)) + .WillOnce(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })); + } + + void expect_journaler_update_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, update_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_open_image(MockOpenImageRequest &mock_open_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_open_image_request, construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_image_request, send()) + .WillOnce(Invoke([this, &mock_open_image_request, &mock_image_ctx, r]() { + *mock_open_image_request.image_ctx = &mock_image_ctx; + m_threads->work_queue->queue(mock_open_image_request.on_finish, r); + })); + } + + void expect_open_local_image(MockOpenLocalImageRequest &mock_open_local_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx *mock_image_ctx, int r) { + EXPECT_CALL(mock_open_local_image_request, + construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_local_image_request, send()) + .WillOnce(Invoke([this, &mock_open_local_image_request, mock_image_ctx, r]() { + *mock_open_local_image_request.image_ctx = mock_image_ctx; + m_threads->work_queue->queue(mock_open_local_image_request.on_finish, + r); + })); + } + + void expect_close_image(MockCloseImageRequest &mock_close_image_request, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_close_image_request, construct(&mock_image_ctx)); + EXPECT_CALL(mock_close_image_request, send()) + .WillOnce(Invoke([this, &mock_close_image_request, r]() { + *mock_close_image_request.image_ctx = nullptr; + m_threads->work_queue->queue(mock_close_image_request.on_finish, r); + })); + } + + void expect_is_primary(MockIsPrimaryRequest &mock_is_primary_request, + bool primary, int r) { + EXPECT_CALL(mock_is_primary_request, send()) + .WillOnce(Invoke([this, &mock_is_primary_request, primary, r]() { + *mock_is_primary_request.primary = primary; + m_threads->work_queue->queue(mock_is_primary_request.on_finish, r); + })); + } + + void expect_journal_get_tag_tid(librbd::MockJournal &mock_journal, + uint64_t tag_tid) { + EXPECT_CALL(mock_journal, get_tag_tid()).WillOnce(Return(tag_tid)); + } + + void expect_journal_get_tag_data(librbd::MockJournal &mock_journal, + const librbd::journal::TagData &tag_data) { + EXPECT_CALL(mock_journal, get_tag_data()).WillOnce(Return(tag_data)); + } + + void expect_is_resync_requested(librbd::MockJournal &mock_journal, + bool do_resync, int r) { + EXPECT_CALL(mock_journal, is_resync_requested(_)) + .WillOnce(DoAll(SetArgPointee<0>(do_resync), + Return(r))); + } + + void expect_create_image(MockCreateImageRequest &mock_create_image_request, + const std::string &image_id, int r) { + EXPECT_CALL(mock_create_image_request, construct(image_id)); + EXPECT_CALL(mock_create_image_request, send()) + .WillOnce(Invoke([this, &mock_create_image_request, r]() { + m_threads->work_queue->queue(mock_create_image_request.on_finish, r); + })); + } + + void expect_image_sync(MockImageSync &mock_image_sync, int r) { + EXPECT_CALL(mock_image_sync, get()); + EXPECT_CALL(mock_image_sync, send()) + .WillOnce(Invoke([this, &mock_image_sync, r]() { + m_threads->work_queue->queue(mock_image_sync.on_finish, r); + })); + EXPECT_CALL(mock_image_sync, put()); + } + + bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { + bufferlist bl; + encode(tag_data, bl); + return bl; + } + + MockBootstrapRequest *create_request(MockThreads* mock_threads, + MockInstanceWatcher *mock_instance_watcher, + ::journal::MockJournaler &mock_journaler, + const std::string &local_image_id, + const std::string &remote_image_id, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &remote_mirror_uuid, + cls::journal::ClientState *client_state, + librbd::journal::MirrorPeerClientMeta *mirror_peer_client_meta, + Context *on_finish) { + return new MockBootstrapRequest(mock_threads, m_local_io_ctx, + m_remote_io_ctx, + mock_instance_watcher, + &m_local_test_image_ctx, + local_image_id, + remote_image_id, + global_image_id, + local_mirror_uuid, + remote_mirror_uuid, + &mock_journaler, + client_state, mirror_peer_client_meta, + on_finish, &m_do_resync); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx = nullptr; + librbd::MockTestImageCtx *m_local_test_image_ctx = nullptr; + bool m_do_resync; +}; + +TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, false, 0); + + // switch the state to replaying + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + client_data.client_meta = mirror_peer_client_meta; + expect_journaler_update_client(mock_journaler, client_data, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(-EREMOTEIO, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteNotTagOwner) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, false, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 344, 0}); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_local_image_ctx, 0); + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(-EREMOTEIO, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, false, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 2, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 4, 369})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, true, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::ORPHAN_MIRROR_UUID, + "remote mirror uuid", true, 4, 1}); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})}, + {5, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 4, 1})}, + {6, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 5, 1})}, + {7, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 6, 1})}, + {8, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 7, 1})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, true, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 346); + expect_journal_get_tag_data(mock_journal, + {librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 345, 1}); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({"local mirror uuid", "local mirror uuid", + true, 344, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + "local mirror uuid", true, 345, 1})}, + {4, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 3, 1})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, true, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::ORPHAN_MIRROR_UUID, + true, 344, 0}); + + // remote demotion / promotion event + Tags tags = { + {2, 123, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 1, 99})}, + {3, 123, encode_tag_data({librbd::Journal<>::ORPHAN_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 2, 1})} + }; + expect_journaler_get_tags(mock_journaler, 123, tags, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_local_image_ctx, 0); + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(-EEXIST, ctx.wait()); + ASSERT_EQ(NULL, m_local_test_image_ctx); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, true, 0); + + // open the local image + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + + // resync is requested + expect_is_resync_requested(mock_journal, true, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{ + mock_local_image_ctx.id}; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, + mock_local_image_ctx.id, mock_remote_image_ctx.id, "global image id", + "local mirror uuid", "remote mirror uuid", &client_state, + &mirror_peer_client_meta, &ctx); + m_do_resync = false; + request->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(m_do_resync); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, true, 0); + + // update client state back to syncing + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + librbd::util::s_image_id = mock_local_image_ctx.id; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + client_data.client_meta = mirror_peer_client_meta; + client.data.clear(); + encode(client_data, client.data); + expect_journaler_update_client(mock_journaler, client_data, 0); + + // create the local image + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, mock_local_image_ctx.id, 0); + + // open the local image + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // sync the remote image to the local image + MockImageSync mock_image_sync; + expect_image_sync(mock_image_sync, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + mirror_peer_client_meta.image_id = ""; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, "", + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, true, 0); + + // open the missing local image + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + "missing image id", nullptr, -ENOENT); + + // re-register the client + expect_journaler_unregister_client(mock_journaler, 0); + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = ""; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + client_data.client_meta = mirror_peer_client_meta; + expect_journaler_register_client(mock_journaler, client_data, 0); + + // test if remote image is primary + expect_is_primary(mock_is_primary_request, true, 0); + + // update client state back to syncing + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + librbd::util::s_image_id = mock_local_image_ctx.id; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + client_data.client_meta = mirror_peer_client_meta; + client.data.clear(); + encode(client_data, client.data); + expect_journaler_update_client(mock_journaler, client_data, 0); + + // create the missing local image + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, mock_local_image_ctx.id, 0); + + // open the local image + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // sync the remote image to the local image + MockImageSync mock_image_sync; + expect_image_sync(mock_image_sync, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + mirror_peer_client_meta.image_id = "missing image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, "missing image id", + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerBootstrapRequest, LocalImageIdCollision) { + create_local_image(); + + InSequence seq; + + // lookup remote image tag class + cls::journal::Client client; + librbd::journal::ClientData client_data{ + librbd::journal::ImageClientMeta{123}}; + encode(client_data, client.data); + ::journal::MockJournaler mock_journaler; + expect_journaler_get_client(mock_journaler, + librbd::Journal<>::IMAGE_CLIENT_ID, + client, 0); + + // open the remote image + librbd::MockJournal mock_journal; + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockOpenImageRequest mock_open_image_request; + expect_open_image(mock_open_image_request, m_remote_io_ctx, + mock_remote_image_ctx.id, mock_remote_image_ctx, 0); + + // test if remote image is primary + MockIsPrimaryRequest mock_is_primary_request; + expect_is_primary(mock_is_primary_request, true, 0); + + // update client state back to syncing + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + mock_local_image_ctx.journal = &mock_journal; + + librbd::util::s_image_id = mock_local_image_ctx.id; + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = mock_local_image_ctx.id; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + client_data.client_meta = mirror_peer_client_meta; + client.data.clear(); + encode(client_data, client.data); + expect_journaler_update_client(mock_journaler, client_data, 0); + + // create the local image + MockCreateImageRequest mock_create_image_request; + expect_create_image(mock_create_image_request, mock_local_image_ctx.id, + -EBADF); + + expect_journaler_update_client(mock_journaler, client_data, 0); + expect_create_image(mock_create_image_request, mock_local_image_ctx.id, 0); + + // open the local image + MockOpenLocalImageRequest mock_open_local_image_request; + expect_open_local_image(mock_open_local_image_request, m_local_io_ctx, + mock_local_image_ctx.id, &mock_local_image_ctx, 0); + expect_is_resync_requested(mock_journal, false, 0); + + expect_journal_get_tag_tid(mock_journal, 345); + expect_journal_get_tag_data(mock_journal, {"remote mirror uuid"}); + + // sync the remote image to the local image + MockImageSync mock_image_sync; + expect_image_sync(mock_image_sync, 0); + + MockCloseImageRequest mock_close_image_request; + expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockInstanceWatcher mock_instance_watcher; + cls::journal::ClientState client_state = cls::journal::CLIENT_STATE_CONNECTED; + mirror_peer_client_meta.image_id = ""; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + MockBootstrapRequest *request = create_request( + &mock_threads, &mock_instance_watcher, mock_journaler, "", + mock_remote_image_ctx.id, "global image id", "local mirror uuid", + "remote mirror uuid", &client_state, &mirror_peer_client_meta, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc new file mode 100644 index 00000000..5c446014 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_CreateImageRequest.cc @@ -0,0 +1,758 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenImageRequest.h" +#include "tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h" +#include "librbd/image/CreateRequest.h" +#include "librbd/image/CloneRequest.h" +#include "tools/rbd_mirror/Threads.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace image { + +template<> +struct CreateRequest<librbd::MockTestImageCtx> { + static CreateRequest *s_instance; + Context *on_finish = nullptr; + + static CreateRequest *create(const ConfigProxy& config, IoCtx &ioctx, + const std::string &imgname, + const std::string &imageid, uint64_t size, + const librbd::ImageOptions &image_options, + const std::string &non_primary_global_image_id, + const std::string &primary_mirror_uuid, + bool skip_mirror_enable, + MockContextWQ *op_work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + EXPECT_FALSE(non_primary_global_image_id.empty()); + EXPECT_FALSE(primary_mirror_uuid.empty()); + EXPECT_FALSE(skip_mirror_enable); + s_instance->on_finish = on_finish; + s_instance->construct(ioctx); + return s_instance; + } + + CreateRequest() { + s_instance = this; + } + + ~CreateRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); + MOCK_METHOD1(construct, void(librados::IoCtx &ioctx)); +}; + +CreateRequest<librbd::MockTestImageCtx>* + CreateRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct CloneRequest<librbd::MockTestImageCtx> { + static CloneRequest *s_instance; + Context *on_finish = nullptr; + + static CloneRequest *create(ConfigProxy& config, IoCtx &p_ioctx, + const std::string &p_id, + const std::string &p_snap_name, + uint64_t p_snap_id, + IoCtx &c_ioctx, const std::string &c_name, + const std::string &c_id, ImageOptions c_options, + const std::string &non_primary_global_image_id, + const std::string &primary_mirror_uuid, + MockContextWQ *op_work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + s_instance->construct(); + return s_instance; + } + + CloneRequest() { + s_instance = this; + } + + ~CloneRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); + MOCK_METHOD0(construct, void()); +}; + +CloneRequest<librbd::MockTestImageCtx>* + CloneRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image + +namespace journal { + +template <> +struct TypeTraits<MockTestImageCtx> { + typedef ::journal::MockJournalerProxy Journaler; +}; + +} // namespace journal +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +namespace image_replayer { + +template<> +struct CloseImageRequest<librbd::MockTestImageCtx> { + static CloseImageRequest* s_instance; + Context *on_finish = nullptr; + + static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->construct(*image_ctx); + s_instance->on_finish = on_finish; + return s_instance; + } + + CloseImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~CloseImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD1(construct, void(librbd::MockTestImageCtx *image_ctx)); + MOCK_METHOD0(send, void()); +}; + +template<> +struct OpenImageRequest<librbd::MockTestImageCtx> { + static OpenImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static OpenImageRequest* create(librados::IoCtx &io_ctx, + librbd::MockTestImageCtx **image_ctx, + const std::string &image_id, + bool read_only, Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + s_instance->construct(io_ctx, image_id); + return s_instance; + } + + OpenImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~OpenImageRequest() { + s_instance = nullptr; + } + + MOCK_METHOD2(construct, void(librados::IoCtx &io_ctx, + const std::string &image_id)); + MOCK_METHOD0(send, void()); +}; + +CloseImageRequest<librbd::MockTestImageCtx>* + CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +OpenImageRequest<librbd::MockTestImageCtx>* + OpenImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.cc" +template class rbd::mirror::image_replayer::CreateImageRequest<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +MATCHER_P(IsSameIoCtx, io_ctx, "") { + return &get_mock_io_ctx(arg) == &get_mock_io_ctx(*io_ctx); +} + +class TestMockImageReplayerCreateImageRequest : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef librbd::image::CreateRequest<librbd::MockTestImageCtx> MockCreateRequest; + typedef librbd::image::CloneRequest<librbd::MockTestImageCtx> MockCloneRequest; + typedef CreateImageRequest<librbd::MockTestImageCtx> MockCreateImageRequest; + typedef OpenImageRequest<librbd::MockTestImageCtx> MockOpenImageRequest; + typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void snap_create(librbd::ImageCtx *image_ctx, const std::string &snap_name) { + ASSERT_EQ(0, image_ctx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + snap_name.c_str())); + ASSERT_EQ(0, image_ctx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + snap_name.c_str())); + ASSERT_EQ(0, image_ctx->state->refresh()); + } + + int clone_image(librbd::ImageCtx *parent_image_ctx, + const std::string &snap_name, const std::string &clone_name) { + snap_create(parent_image_ctx, snap_name); + + int order = 0; + return librbd::clone(m_remote_io_ctx, parent_image_ctx->name.c_str(), + snap_name.c_str(), m_remote_io_ctx, + clone_name.c_str(), parent_image_ctx->features, + &order, 0, 0); + } + + void expect_create_image(MockCreateRequest &mock_create_request, + librados::IoCtx &ioctx, int r) { + EXPECT_CALL(mock_create_request, construct(IsSameIoCtx(&ioctx))); + EXPECT_CALL(mock_create_request, send()) + .WillOnce(Invoke([this, &mock_create_request, r]() { + m_threads->work_queue->queue(mock_create_request.on_finish, r); + })); + } + + void expect_ioctx_create(librados::IoCtx &io_ctx) { + librados::MockTestMemIoCtxImpl &io_ctx_impl = get_mock_io_ctx(io_ctx); + EXPECT_CALL(*get_mock_io_ctx(io_ctx).get_mock_rados_client(), create_ioctx(_, _)) + .WillOnce(DoAll(GetReference(&io_ctx_impl), + Return(&get_mock_io_ctx(io_ctx)))); + } + + void expect_get_parent_global_image_id(librados::IoCtx &io_ctx, + const std::string &global_id, int r) { + cls::rbd::MirrorImage mirror_image; + mirror_image.global_image_id = global_id; + + bufferlist bl; + encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_mirror_image_get_image_id(librados::IoCtx &io_ctx, + const std::string &image_id, int r) { + bufferlist bl; + encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get_image_id"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_open_image(MockOpenImageRequest &mock_open_image_request, + librados::IoCtx &io_ctx, const std::string &image_id, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_open_image_request, construct(IsSameIoCtx(&io_ctx), image_id)); + EXPECT_CALL(mock_open_image_request, send()) + .WillOnce(Invoke([this, &mock_open_image_request, &mock_image_ctx, r]() { + *mock_open_image_request.image_ctx = &mock_image_ctx; + m_threads->work_queue->queue(mock_open_image_request.on_finish, r); + })); + } + + void expect_test_op_features(librbd::MockTestImageCtx& mock_image_ctx, + bool enabled) { + EXPECT_CALL(mock_image_ctx, + test_op_features(RBD_OPERATION_FEATURE_CLONE_CHILD)) + .WillOnce(Return(enabled)); + } + + void expect_clone_image(MockCloneRequest &mock_clone_request, + int r) { + EXPECT_CALL(mock_clone_request, construct()); + EXPECT_CALL(mock_clone_request, send()) + .WillOnce(Invoke([this, &mock_clone_request, r]() { + m_threads->work_queue->queue(mock_clone_request.on_finish, r); + })); + } + + void expect_close_image(MockCloseImageRequest &mock_close_image_request, + librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(mock_close_image_request, construct(&mock_image_ctx)); + EXPECT_CALL(mock_close_image_request, send()) + .WillOnce(Invoke([this, &mock_close_image_request, r]() { + m_threads->work_queue->queue(mock_close_image_request.on_finish, r); + })); + } + + void expect_mirror_uuid_get(librados::IoCtx &io_ctx, + const std::string &mirror_uuid, int r) { + bufferlist bl; + encode(mirror_uuid, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_journaler_get_client(journal::MockJournaler& mock_journaler, + const std::string& client_id, + librbd::journal::MirrorPeerState state, + int r) { + EXPECT_CALL(mock_journaler, construct()); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.state = state; + + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + + cls::journal::Client client; + encode(client_data, client.data); + + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + MockCreateImageRequest *create_request(MockThreads* mock_threads, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + librbd::MockTestImageCtx &mock_remote_image_ctx, + Context *on_finish) { + return new MockCreateImageRequest(mock_threads, m_local_io_ctx, + global_image_id, remote_mirror_uuid, + local_image_name, local_image_id, + &mock_remote_image_ctx, on_finish); + } + + librbd::ImageCtx *m_remote_image_ctx; +}; + +TEST_F(TestMockImageReplayerCreateImageRequest, Create) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockCreateRequest mock_create_request; + + InSequence seq; + expect_create_image(mock_create_request, m_local_io_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_image_ctx, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CreateError) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + MockCreateRequest mock_create_request; + + InSequence seq; + expect_create_image(mock_create_request, m_local_io_ctx, -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_image_ctx, &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, Clone) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockCloneRequest mock_clone_request; + MockOpenImageRequest mock_open_image_request; + MockCloseImageRequest mock_close_image_request; + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_test_op_features(mock_remote_clone_image_ctx, false); + expect_clone_image(mock_clone_request, 0); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneParentMirrorUuidGetError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", -EPERM); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, -EPERM); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetRemoteParentClientStateSyncing) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_SYNCING, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetGlobalImageIdError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", -ENOENT); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneGetLocalParentImageIdError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", -ENOENT); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneOpenRemoteParentError) { + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockOpenImageRequest mock_open_image_request; + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, + -ENOENT); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneError) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockCloneRequest mock_clone_request; + MockOpenImageRequest mock_open_image_request; + MockCloseImageRequest mock_close_image_request; + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_test_op_features(mock_remote_clone_image_ctx, false); + expect_clone_image(mock_clone_request, -EINVAL); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, 0); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerCreateImageRequest, CloneRemoteParentCloseError) { + librbd::RBD rbd; + librbd::ImageCtx *local_image_ctx; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &local_image_ctx)); + snap_create(local_image_ctx, "snap"); + + std::string clone_image_name = get_temp_image_name(); + ASSERT_EQ(0, clone_image(m_remote_image_ctx, "snap", clone_image_name)); + + librbd::ImageCtx *remote_clone_image_ctx; + ASSERT_EQ(0, open_image(m_remote_io_ctx, clone_image_name, + &remote_clone_image_ctx)); + + librbd::MockTestImageCtx mock_remote_parent_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_remote_clone_image_ctx(*remote_clone_image_ctx); + MockCloneRequest mock_clone_request; + MockOpenImageRequest mock_open_image_request; + MockCloseImageRequest mock_close_image_request; + journal::MockJournaler mock_remote_journaler; + + InSequence seq; + expect_ioctx_create(m_remote_io_ctx); + expect_ioctx_create(m_local_io_ctx); + expect_mirror_uuid_get(m_local_io_ctx, "local parent uuid", 0); + expect_journaler_get_client( + mock_remote_journaler, "local parent uuid", + librbd::journal::MIRROR_PEER_STATE_REPLAYING, 0); + expect_get_parent_global_image_id(m_remote_io_ctx, "global uuid", 0); + expect_mirror_image_get_image_id(m_local_io_ctx, "local parent id", 0); + + expect_open_image(mock_open_image_request, m_remote_io_ctx, + m_remote_image_ctx->id, mock_remote_parent_image_ctx, 0); + expect_test_op_features(mock_remote_clone_image_ctx, false); + expect_clone_image(mock_clone_request, 0); + expect_close_image(mock_close_image_request, mock_remote_parent_image_ctx, + -EINVAL); + + C_SaferCond ctx; + MockThreads mock_threads(m_threads); + MockCreateImageRequest *request = create_request(&mock_threads, "global uuid", + "remote uuid", "image name", + "101241a7c4c9", + mock_remote_clone_image_ctx, + &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_EventPreprocessor.cc b/src/test/rbd_mirror/image_replayer/test_mock_EventPreprocessor.cc new file mode 100644 index 00000000..9e02a82f --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_EventPreprocessor.cc @@ -0,0 +1,265 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/image_replayer/EventPreprocessor.h" +#include "tools/rbd_mirror/Threads.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/EventPreprocessor.cc" +template class rbd::mirror::image_replayer::EventPreprocessor<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using testing::_; +using testing::WithArg; + +class TestMockImageReplayerEventPreprocessor : public TestMockFixture { +public: + typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_image_refresh(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.state, refresh(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_update_client(journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, update_client(_, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + + librbd::ImageCtx *m_local_image_ctx; + librbd::journal::MirrorPeerClientMeta m_client_meta; + +}; + +TEST_F(TestMockImageReplayerEventPreprocessor, IsNotRequired) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + ASSERT_FALSE(event_preprocessor.is_required(event_entry)); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, IsRequiredSnapMapPrune) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + m_client_meta.snap_seqs = {{1, 2}, {3, 4}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + ASSERT_TRUE(event_preprocessor.is_required(event_entry)); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, IsRequiredSnapRename) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::SnapRenameEvent{}}; + ASSERT_TRUE(event_preprocessor.is_required(event_entry)); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, PreprocessSnapMapPrune) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + expect_update_client(mock_remote_journaler, 0); + + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + m_client_meta.snap_seqs = {{1, 2}, {3, 4}, {5, 6}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(0, ctx.wait()); + + librbd::SnapSeqs expected_snap_seqs = {{5, 6}}; + ASSERT_EQ(expected_snap_seqs, m_client_meta.snap_seqs); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, PreprocessSnapRename) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + expect_update_client(mock_remote_journaler, 0); + + mock_local_image_ctx.snap_ids = {{{cls::rbd::UserSnapshotNamespace(), "snap"}, 6}}; + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(0, ctx.wait()); + + librbd::SnapSeqs expected_snap_seqs = {{5, 6}}; + ASSERT_EQ(expected_snap_seqs, m_client_meta.snap_seqs); + + librbd::journal::SnapRenameEvent *event = + boost::get<librbd::journal::SnapRenameEvent>(&event_entry.event); + ASSERT_EQ(6U, event->snap_id); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, PreprocessSnapRenameMissing) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + librbd::journal::SnapRenameEvent *event = + boost::get<librbd::journal::SnapRenameEvent>(&event_entry.event); + ASSERT_EQ(CEPH_NOSNAP, event->snap_id); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, PreprocessSnapRenameKnown) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + m_client_meta.snap_seqs = {{5, 6}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(0, ctx.wait()); + + librbd::SnapSeqs expected_snap_seqs = {{5, 6}}; + ASSERT_EQ(expected_snap_seqs, m_client_meta.snap_seqs); + + librbd::journal::SnapRenameEvent *event = + boost::get<librbd::journal::SnapRenameEvent>(&event_entry.event); + ASSERT_EQ(6U, event->snap_id); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, PreprocessRefreshError) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, -EINVAL); + + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{librbd::journal::RenameEvent{}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerEventPreprocessor, PreprocessClientUpdateError) { + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + ::journal::MockJournaler mock_remote_journaler; + + expect_image_refresh(mock_local_image_ctx, 0); + expect_update_client(mock_remote_journaler, -EINVAL); + + mock_local_image_ctx.snap_ids = {{{cls::rbd::UserSnapshotNamespace(), "snap"}, 6}}; + mock_local_image_ctx.snap_info = { + {6, librbd::SnapInfo{"snap", cls::rbd::UserSnapshotNamespace(), 0U, {}, 0U, 0U, utime_t()}}}; + MockEventPreprocessor event_preprocessor(mock_local_image_ctx, + mock_remote_journaler, + "local mirror uuid", + &m_client_meta, + m_threads->work_queue); + + librbd::journal::EventEntry event_entry{ + librbd::journal::SnapRenameEvent{0, 5, "snap", "new_snap"}}; + C_SaferCond ctx; + event_preprocessor.preprocess(&event_entry, &ctx); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_GetMirrorImageIdRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_GetMirrorImageIdRequest.cc new file mode 100644 index 00000000..40007dfd --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_GetMirrorImageIdRequest.cc @@ -0,0 +1,106 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageReplayerGetMirrorImageIdRequest : public TestMockFixture { +public: + typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest; + + void expect_mirror_image_get_image_id(librados::IoCtx &io_ctx, + const std::string &image_id, int r) { + bufferlist bl; + encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get_image_id"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + +}; + +TEST_F(TestMockImageReplayerGetMirrorImageIdRequest, Success) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "image id", 0); + + std::string image_id; + C_SaferCond ctx; + auto req = MockGetMirrorImageIdRequest::create(m_local_io_ctx, + "global image id", + &image_id, &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("image id"), image_id); +} + +TEST_F(TestMockImageReplayerGetMirrorImageIdRequest, MirrorImageIdDNE) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "", -ENOENT); + + std::string image_id; + C_SaferCond ctx; + auto req = MockGetMirrorImageIdRequest::create(m_local_io_ctx, + "global image id", + &image_id, &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerGetMirrorImageIdRequest, MirrorImageIdError) { + InSequence seq; + expect_mirror_image_get_image_id(m_local_io_ctx, "", -EINVAL); + + std::string image_id; + C_SaferCond ctx; + auto req = MockGetMirrorImageIdRequest::create(m_local_io_ctx, + "global image id", + &image_id, &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc new file mode 100644 index 00000000..64e0697e --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc @@ -0,0 +1,264 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <> +struct GetMirrorImageIdRequest<librbd::MockTestImageCtx> { + static GetMirrorImageIdRequest* s_instance; + std::string* image_id = nullptr; + Context* on_finish = nullptr; + + static GetMirrorImageIdRequest* create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + std::string* image_id, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_id = image_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetMirrorImageIdRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +GetMirrorImageIdRequest<librbd::MockTestImageCtx>* GetMirrorImageIdRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageReplayerPrepareLocalImageRequest : public TestMockFixture { +public: + typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest; + typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest; + + void expect_get_mirror_image_id(MockGetMirrorImageIdRequest& mock_get_mirror_image_id_request, + const std::string& image_id, int r) { + EXPECT_CALL(mock_get_mirror_image_id_request, send()) + .WillOnce(Invoke([&mock_get_mirror_image_id_request, image_id, r]() { + *mock_get_mirror_image_id_request.image_id = image_id; + mock_get_mirror_image_id_request.on_finish->complete(r); + })); + } + + void expect_dir_get_name(librados::IoCtx &io_ctx, + const std::string &image_name, int r) { + bufferlist bl; + encode(image_name, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_DIRECTORY, _, StrEq("rbd"), StrEq("dir_get_name"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_mirror_image_get(librados::IoCtx &io_ctx, + cls::rbd::MirrorImageState state, + const std::string &global_id, int r) { + cls::rbd::MirrorImage mirror_image; + mirror_image.state = state; + mirror_image.global_image_id = global_id; + + bufferlist bl; + encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_get_tag_owner(librbd::MockJournal &mock_journal, + const std::string &local_image_id, + const std::string &tag_owner, int r) { + EXPECT_CALL(mock_journal, get_tag_owner(local_image_id, _, _, _)) + .WillOnce(WithArgs<1, 3>(Invoke([tag_owner, r](std::string *owner, Context *on_finish) { + *owner = tag_owner; + on_finish->complete(r); + }))); + } + +}; + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, Success) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + expect_mirror_image_get(m_local_io_ctx, cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + "global image id", 0); + + librbd::MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "local image id", "remote mirror uuid", 0); + + std::string local_image_id; + std::string local_image_name; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &local_image_name, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("local image id"), local_image_id); + ASSERT_EQ(std::string("local image name"), local_image_name); + ASSERT_EQ(std::string("remote mirror uuid"), tag_owner); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageIdError) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "", -EINVAL); + + std::string local_image_id; + std::string local_image_name; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &local_image_name, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, DirGetNameError) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "", -ENOENT); + + std::string local_image_id; + std::string local_image_name; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &local_image_name, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageError) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + expect_mirror_image_get(m_local_io_ctx, cls::rbd::MIRROR_IMAGE_STATE_DISABLED, + "", -EINVAL); + + std::string local_image_id; + std::string local_image_name; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &local_image_name, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, TagOwnerError) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + expect_mirror_image_get(m_local_io_ctx, cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + "global image id", 0); + + librbd::MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "local image id", "remote mirror uuid", + -ENOENT); + + std::string local_image_id; + std::string local_image_name; + std::string tag_owner; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_id, + &local_image_name, + &tag_owner, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc new file mode 100644 index 00000000..1b957ed1 --- /dev/null +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareRemoteImageRequest.cc @@ -0,0 +1,396 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<MockTestImageCtx> { + typedef ::journal::MockJournalerProxy Journaler; +}; + +} // namespace journal +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +namespace image_replayer { + +template <> +struct GetMirrorImageIdRequest<librbd::MockTestImageCtx> { + static GetMirrorImageIdRequest* s_instance; + std::string* image_id = nullptr; + Context* on_finish = nullptr; + + static GetMirrorImageIdRequest* create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + std::string* image_id, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_id = image_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + GetMirrorImageIdRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +GetMirrorImageIdRequest<librbd::MockTestImageCtx>* GetMirrorImageIdRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageReplayerPrepareRemoteImageRequest : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest; + typedef GetMirrorImageIdRequest<librbd::MockTestImageCtx> MockGetMirrorImageIdRequest; + + void expect_get_mirror_image_id(MockGetMirrorImageIdRequest& mock_get_mirror_image_id_request, + const std::string& image_id, int r) { + EXPECT_CALL(mock_get_mirror_image_id_request, send()) + .WillOnce(Invoke([&mock_get_mirror_image_id_request, image_id, r]() { + *mock_get_mirror_image_id_request.image_id = image_id; + mock_get_mirror_image_id_request.on_finish->complete(r); + })); + } + + void expect_mirror_uuid_get(librados::IoCtx &io_ctx, + const std::string &mirror_uuid, int r) { + bufferlist bl; + encode(mirror_uuid, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_journaler_get_client(::journal::MockJournaler &mock_journaler, + const std::string &client_id, + cls::journal::Client &client, int r) { + EXPECT_CALL(mock_journaler, get_client(StrEq(client_id), _, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([client](cls::journal::Client *out_client) { + *out_client = client; + })), + WithArg<2>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + })))); + } + + void expect_journaler_register_client(::journal::MockJournaler &mock_journaler, + const librbd::journal::ClientData &client_data, + int r) { + bufferlist bl; + encode(client_data, bl); + + EXPECT_CALL(mock_journaler, register_client(ContentsEqual(bl), _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *on_finish) { + m_threads->work_queue->queue(on_finish, r); + }))); + } +}; + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, Success) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + cls::journal::Client client; + client.state = cls::journal::CLIENT_STATE_DISCONNECTED; + encode(client_data, client.data); + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, 0); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", {}, + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_DISCONNECTED, client_state); + delete remote_journaler; +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, SuccessNotRegistered) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, 0); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", {}, + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler != nullptr); + ASSERT_EQ(cls::journal::CLIENT_STATE_CONNECTED, client_state); + delete remote_journaler; +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorUuidError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "", -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "", {}, + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string(""), remote_mirror_uuid); + ASSERT_TRUE(remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, MirrorImageIdError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "", -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "", {}, + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_TRUE(remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, GetClientError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", {}, + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareRemoteImageRequest, RegisterClientError) { + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + + InSequence seq; + expect_mirror_uuid_get(m_remote_io_ctx, "remote mirror uuid", 0); + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, + "remote image id", 0); + + EXPECT_CALL(mock_remote_journaler, construct()); + + cls::journal::Client client; + expect_journaler_get_client(mock_remote_journaler, "local mirror uuid", + client, -ENOENT); + + librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta; + mirror_peer_client_meta.image_id = "local image id"; + mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + librbd::journal::ClientData client_data{mirror_peer_client_meta}; + expect_journaler_register_client(mock_remote_journaler, client_data, -EINVAL); + + std::string remote_mirror_uuid; + std::string remote_image_id; + journal::MockJournalerProxy *remote_journaler = nullptr; + cls::journal::ClientState client_state; + librbd::journal::MirrorPeerClientMeta client_meta; + C_SaferCond ctx; + auto req = MockPrepareRemoteImageRequest::create(&mock_threads, + m_remote_io_ctx, + "global image id", + "local mirror uuid", + "local image id", {}, + &remote_mirror_uuid, + &remote_image_id, + &remote_journaler, + &client_state, &client_meta, + &ctx); + req->send(); + + ASSERT_EQ(-EINVAL, ctx.wait()); + ASSERT_EQ(std::string("remote mirror uuid"), remote_mirror_uuid); + ASSERT_EQ(std::string("remote image id"), remote_image_id); + ASSERT_TRUE(remote_journaler == nullptr); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_sync/test_mock_SyncPointCreateRequest.cc b/src/test/rbd_mirror/image_sync/test_mock_SyncPointCreateRequest.cc new file mode 100644 index 00000000..37556257 --- /dev/null +++ b/src/test/rbd_mirror/image_sync/test_mock_SyncPointCreateRequest.cc @@ -0,0 +1,171 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc" +template class rbd::mirror::image_sync::SyncPointCreateRequest<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace image_sync { + +using ::testing::_; +using ::testing::InSequence; +using ::testing::WithArg; + +class TestMockImageSyncSyncPointCreateRequest : public TestMockFixture { +public: + typedef SyncPointCreateRequest<librbd::MockTestImageCtx> MockSyncPointCreateRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void expect_update_client(journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, update_client(_, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + + void expect_image_refresh(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.state, refresh(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_snap_create(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.operations, snap_create(_, _, _)) + .WillOnce(WithArg<2>(CompleteContext(r))); + } + + MockSyncPointCreateRequest *create_request(librbd::MockTestImageCtx &mock_remote_image_ctx, + journal::MockJournaler &mock_journaler, + Context *ctx) { + return new MockSyncPointCreateRequest(&mock_remote_image_ctx, "uuid", + &mock_journaler, &m_client_meta, ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::journal::MirrorPeerClientMeta m_client_meta; +}; + +TEST_F(TestMockImageSyncSyncPointCreateRequest, Success) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_update_client(mock_journaler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(1U, m_client_meta.sync_points.size()); +} + +TEST_F(TestMockImageSyncSyncPointCreateRequest, ResyncSuccess) { + m_client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "start snap", + "", + boost::none); + auto sync_point = m_client_meta.sync_points.front(); + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_update_client(mock_journaler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(2U, m_client_meta.sync_points.size()); + ASSERT_EQ(sync_point, m_client_meta.sync_points.front()); + ASSERT_EQ("start snap", m_client_meta.sync_points.back().from_snap_name); +} + +TEST_F(TestMockImageSyncSyncPointCreateRequest, SnapshotExists) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_update_client(mock_journaler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, -EEXIST); + expect_update_client(mock_journaler, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_snap_create(mock_remote_image_ctx, 0); + expect_image_refresh(mock_remote_image_ctx, 0); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(1U, m_client_meta.sync_points.size()); +} + +TEST_F(TestMockImageSyncSyncPointCreateRequest, ClientUpdateError) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_update_client(mock_journaler, -EINVAL); + + C_SaferCond ctx; + MockSyncPointCreateRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); + + ASSERT_TRUE(m_client_meta.sync_points.empty()); +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/image_sync/test_mock_SyncPointPruneRequest.cc b/src/test/rbd_mirror/image_sync/test_mock_SyncPointPruneRequest.cc new file mode 100644 index 00000000..d230944e --- /dev/null +++ b/src/test/rbd_mirror/image_sync/test_mock_SyncPointPruneRequest.cc @@ -0,0 +1,338 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc" +template class rbd::mirror::image_sync::SyncPointPruneRequest<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace image_sync { + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageSyncSyncPointPruneRequest : public TestMockFixture { +public: + typedef SyncPointPruneRequest<librbd::MockTestImageCtx> MockSyncPointPruneRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void expect_update_client(journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, update_client(_, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + + void expect_get_snap_id(librbd::MockTestImageCtx &mock_remote_image_ctx, + const std::string &snap_name, uint64_t snap_id) { + EXPECT_CALL(mock_remote_image_ctx, get_snap_id(_, StrEq(snap_name))) + .WillOnce(Return(snap_id)); + } + + void expect_image_refresh(librbd::MockTestImageCtx &mock_remote_image_ctx, int r) { + EXPECT_CALL(*mock_remote_image_ctx.state, refresh(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_snap_remove(librbd::MockTestImageCtx &mock_remote_image_ctx, + const std::string &snap_name, int r) { + EXPECT_CALL(*mock_remote_image_ctx.operations, snap_remove(_, StrEq(snap_name), _)) + .WillOnce(WithArg<2>(CompleteContext(r))); + } + + MockSyncPointPruneRequest *create_request(librbd::MockTestImageCtx &mock_remote_image_ctx, + journal::MockJournaler &mock_journaler, + bool sync_complete, Context *ctx) { + return new MockSyncPointPruneRequest(&mock_remote_image_ctx, sync_complete, + &mock_journaler, &m_client_meta, ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::journal::MirrorPeerClientMeta m_client_meta; +}; + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncInProgressSuccess) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap1", 123); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_EQ(client_meta, m_client_meta); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, RestartedSyncInProgressSuccess) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap2", + "snap1", boost::none); + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap1", 123); + expect_snap_remove(mock_remote_image_ctx, "snap2", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + client_meta.sync_points.pop_back(); + ASSERT_EQ(client_meta, m_client_meta); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncInProgressMissingSnapSuccess) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap2", + "snap1", + boost::none); + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap1", CEPH_NOSNAP); + expect_snap_remove(mock_remote_image_ctx, "snap2", 0); + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + client_meta.sync_points.clear(); + ASSERT_EQ(client_meta, m_client_meta); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncInProgressUnexpectedFromSnapSuccess) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap2", + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_get_snap_id(mock_remote_image_ctx, "snap2", 124); + expect_snap_remove(mock_remote_image_ctx, "snap2", 0); + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, false, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + + client_meta.sync_points.clear(); + ASSERT_EQ(client_meta, m_client_meta); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SyncCompleteSuccess) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + m_client_meta = client_meta; + ASSERT_EQ(librbd::journal::MIRROR_PEER_STATE_SYNCING, m_client_meta.state); + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(m_client_meta.sync_points.empty()); + ASSERT_EQ(librbd::journal::MIRROR_PEER_STATE_REPLAYING, m_client_meta.state); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, RestartedSyncCompleteSuccess) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap2", + "snap1", + boost::none); + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + client_meta.sync_points.pop_front(); + ASSERT_EQ(client_meta, m_client_meta); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, RestartedCatchUpSyncCompleteSuccess) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap3", + "snap2", + boost::none); + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap2", + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_snap_remove(mock_remote_image_ctx, "snap1", 0); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + client_meta.sync_points.pop_front(); + ASSERT_EQ(client_meta, m_client_meta); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, SnapshotDNE) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_snap_remove(mock_remote_image_ctx, "snap1", -ENOENT); + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, 0); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, true, &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(m_client_meta.sync_points.empty()); +} + +TEST_F(TestMockImageSyncSyncPointPruneRequest, ClientUpdateError) { + librbd::journal::MirrorPeerClientMeta client_meta; + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap2", + "snap1", + boost::none); + client_meta.sync_points.emplace_front(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + m_client_meta = client_meta; + + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + journal::MockJournaler mock_journaler; + + InSequence seq; + expect_image_refresh(mock_remote_image_ctx, 0); + expect_update_client(mock_journaler, -EINVAL); + + C_SaferCond ctx; + MockSyncPointPruneRequest *req = create_request(mock_remote_image_ctx, + mock_journaler, true, &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); + + ASSERT_EQ(client_meta, m_client_meta); +} + +} // namespace image_sync +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/mock/MockContextWQ.h b/src/test/rbd_mirror/mock/MockContextWQ.h new file mode 100644 index 00000000..1c0ee88f --- /dev/null +++ b/src/test/rbd_mirror/mock/MockContextWQ.h @@ -0,0 +1,18 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_MOCK_CONTEXT_WQ_H +#define CEPH_MOCK_CONTEXT_WQ_H + +#include <gmock/gmock.h> + +struct Context; + +struct MockContextWQ { + void queue(Context *ctx) { + queue(ctx, 0); + } + MOCK_METHOD2(queue, void(Context *, int)); +}; + +#endif // CEPH_MOCK_CONTEXT_WQ_H diff --git a/src/test/rbd_mirror/mock/MockSafeTimer.h b/src/test/rbd_mirror/mock/MockSafeTimer.h new file mode 100644 index 00000000..32d58471 --- /dev/null +++ b/src/test/rbd_mirror/mock/MockSafeTimer.h @@ -0,0 +1,16 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_MOCK_SAFE_TIMER_H +#define CEPH_MOCK_SAFE_TIMER_H + +#include <gmock/gmock.h> + +struct Context; + +struct MockSafeTimer { + MOCK_METHOD2(add_event_after, Context*(double, Context*)); + MOCK_METHOD1(cancel_event, bool(Context *)); +}; + +#endif // CEPH_MOCK_SAFE_TIMER_H diff --git a/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc new file mode 100644 index 00000000..afabcdfc --- /dev/null +++ b/src/test/rbd_mirror/pool_watcher/test_mock_RefreshImagesRequest.cc @@ -0,0 +1,116 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h" +#include "include/stringify.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.cc" +template class rbd::mirror::pool_watcher::RefreshImagesRequest<librbd::MockTestImageCtx>; + +namespace rbd { +namespace mirror { +namespace pool_watcher { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockPoolWatcherRefreshImagesRequest : public TestMockFixture { +public: + typedef RefreshImagesRequest<librbd::MockTestImageCtx> MockRefreshImagesRequest; + + void expect_mirror_image_list(librados::IoCtx &io_ctx, + const std::map<std::string, std::string> &ids, + int r) { + bufferlist bl; + encode(ids, bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_list"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + +}; + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, Success) { + InSequence seq; + expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(0, ctx.wait()); + + ImageIds expected_image_ids = {{"global id", "local id"}}; + ASSERT_EQ(expected_image_ids, image_ids); +} + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, LargeDirectory) { + InSequence seq; + std::map<std::string, std::string> mirror_list; + ImageIds expected_image_ids; + for (uint32_t idx = 1; idx <= 1024; ++idx) { + mirror_list.insert(std::make_pair("local id " + stringify(idx), + "global id " + stringify(idx))); + expected_image_ids.insert({{"global id " + stringify(idx), + "local id " + stringify(idx)}}); + } + + expect_mirror_image_list(m_remote_io_ctx, mirror_list, 0); + expect_mirror_image_list(m_remote_io_ctx, {{"local id", "global id"}}, 0); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(0, ctx.wait()); + + expected_image_ids.insert({"global id", "local id"}); + ASSERT_EQ(expected_image_ids, image_ids); +} + +TEST_F(TestMockPoolWatcherRefreshImagesRequest, MirrorImageListError) { + InSequence seq; + expect_mirror_image_list(m_remote_io_ctx, {}, -EINVAL); + + C_SaferCond ctx; + ImageIds image_ids; + MockRefreshImagesRequest *req = new MockRefreshImagesRequest( + m_remote_io_ctx, &image_ids, &ctx); + + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace pool_watcher +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/random_write.cc b/src/test/rbd_mirror/random_write.cc new file mode 100644 index 00000000..8a5423f0 --- /dev/null +++ b/src/test/rbd_mirror/random_write.cc @@ -0,0 +1,213 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/ceph_argparse.h" +#include "common/config.h" +#include "common/debug.h" +#include "common/errno.h" +#include "common/Cond.h" +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "global/global_init.h" +#include <string> +#include <vector> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "random-write: " + +namespace { + +const uint32_t NUM_THREADS = 8; +const uint32_t MAX_IO_SIZE = 24576; +const uint32_t MIN_IO_SIZE = 4; + +void usage() { + std::cout << "usage: ceph_test_rbd_mirror_random_write [options...] \\" << std::endl; + std::cout << " <pool> <image>" << std::endl; + std::cout << std::endl; + std::cout << " pool image pool" << std::endl; + std::cout << " image image to write" << std::endl; + std::cout << std::endl; + std::cout << "options:\n"; + std::cout << " -m monaddress[:port] connect to specified monitor\n"; + std::cout << " --keyring=<path> path to keyring for local cluster\n"; + std::cout << " --log-file=<logfile> file to log debug output\n"; + std::cout << " --debug-rbd-mirror=<log-level>/<memory-level> set rbd-mirror debug level\n"; + generic_server_usage(); +} + +void rbd_bencher_completion(void *c, void *pc); + +struct rbd_bencher { + librbd::Image *image; + Mutex lock; + Cond cond; + int in_flight; + + explicit rbd_bencher(librbd::Image *i) + : image(i), + lock("rbd_bencher::lock"), + in_flight(0) { + } + + bool start_write(int max, uint64_t off, uint64_t len, bufferlist& bl, + int op_flags) { + { + Mutex::Locker l(lock); + if (in_flight >= max) + return false; + in_flight++; + } + librbd::RBD::AioCompletion *c = + new librbd::RBD::AioCompletion((void *)this, rbd_bencher_completion); + image->aio_write2(off, len, bl, c, op_flags); + //cout << "start " << c << " at " << off << "~" << len << std::endl; + return true; + } + + void wait_for(int max) { + Mutex::Locker l(lock); + while (in_flight > max) { + utime_t dur; + dur.set_from_double(.2); + cond.WaitInterval(lock, dur); + } + } + +}; + +void rbd_bencher_completion(void *vc, void *pc) { + librbd::RBD::AioCompletion *c = (librbd::RBD::AioCompletion *)vc; + rbd_bencher *b = static_cast<rbd_bencher *>(pc); + //cout << "complete " << c << std::endl; + int ret = c->get_return_value(); + if (ret != 0) { + cout << "write error: " << cpp_strerror(ret) << std::endl; + exit(ret < 0 ? -ret : ret); + } + b->lock.Lock(); + b->in_flight--; + b->cond.Signal(); + b->lock.Unlock(); + c->release(); +} + +void write_image(librbd::Image &image) { + srand(time(NULL) % (unsigned long) -1); + + uint64_t max_io_bytes = MAX_IO_SIZE * 1024; + bufferptr bp(max_io_bytes); + memset(bp.c_str(), rand() & 0xff, bp.length()); + bufferlist bl; + bl.push_back(bp); + + uint64_t size = 0; + image.size(&size); + ceph_assert(size != 0); + + vector<uint64_t> thread_offset; + uint64_t i; + uint64_t start_pos; + + // disturb all thread's offset, used by seq write + for (i = 0; i < NUM_THREADS; i++) { + start_pos = (rand() % (size / max_io_bytes)) * max_io_bytes; + thread_offset.push_back(start_pos); + } + + uint64_t total_ios = 0; + uint64_t total_bytes = 0; + rbd_bencher b(&image); + while (true) { + b.wait_for(NUM_THREADS - 1); + for (uint32_t i = 0; i < NUM_THREADS; ++i) { + // mostly small writes with a small chance of large writes + uint32_t io_modulo = MIN_IO_SIZE + 1; + if (rand() % 30 == 0) { + io_modulo += MAX_IO_SIZE; + } + + uint32_t io_size = (((rand() % io_modulo) + MIN_IO_SIZE) * 1024); + thread_offset[i] = (rand() % (size / io_size)) * io_size; + if (!b.start_write(NUM_THREADS, thread_offset[i], io_size, bl, + LIBRADOS_OP_FLAG_FADVISE_RANDOM)) { + break; + } + ++i; + + ++total_ios; + total_bytes += io_size; + if (total_ios % 100 == 0) { + std::cout << total_ios << " IOs, " << total_bytes << " bytes" + << std::endl; + } + } + } + b.wait_for(0); +} + +} // anonymous namespace + +int main(int argc, const char **argv) +{ + std::vector<const char*> args; + argv_to_vec(argc, argv, args); + if (args.empty()) { + cerr << argv[0] << ": -h or --help for usage" << std::endl; + exit(1); + } + if (ceph_argparse_need_usage(args)) { + usage(); + exit(0); + } + + auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_MON_CONFIG); + + if (args.size() < 2) { + usage(); + return EXIT_FAILURE; + } + + std::string pool_name = args[0]; + std::string image_name = args[1]; + + common_init_finish(g_ceph_context); + + dout(5) << "connecting to cluster" << dendl; + librados::Rados rados; + librados::IoCtx io_ctx; + librbd::RBD rbd; + librbd::Image image; + int r = rados.init_with_context(g_ceph_context); + if (r < 0) { + derr << "could not initialize RADOS handle" << dendl; + return EXIT_FAILURE; + } + + r = rados.connect(); + if (r < 0) { + derr << "error connecting to local cluster" << dendl; + return EXIT_FAILURE; + } + + r = rados.ioctx_create(pool_name.c_str(), io_ctx); + if (r < 0) { + derr << "error finding local pool " << pool_name << ": " + << cpp_strerror(r) << dendl; + return EXIT_FAILURE; + } + + r = rbd.open(io_ctx, image, image_name.c_str()); + if (r < 0) { + derr << "error opening image " << image_name << ": " + << cpp_strerror(r) << dendl; + return EXIT_FAILURE; + } + + write_image(image); + return EXIT_SUCCESS; +} diff --git a/src/test/rbd_mirror/test_ClusterWatcher.cc b/src/test/rbd_mirror/test_ClusterWatcher.cc new file mode 100644 index 00000000..05ca2c10 --- /dev/null +++ b/src/test/rbd_mirror/test_ClusterWatcher.cc @@ -0,0 +1,254 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#include "include/rados/librados.hpp" +#include "common/Cond.h" +#include "common/errno.h" +#include "common/Mutex.h" +#include "librbd/internal.h" +#include "librbd/api/Mirror.h" +#include "tools/rbd_mirror/ClusterWatcher.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Types.h" +#include "test/rbd_mirror/test_fixture.h" +#include "test/librados/test_cxx.h" +#include "test/librbd/test_support.h" +#include "gtest/gtest.h" +#include <boost/scope_exit.hpp> +#include <iostream> +#include <map> +#include <memory> +#include <set> + +using rbd::mirror::ClusterWatcher; +using rbd::mirror::PeerSpec; +using rbd::mirror::RadosRef; +using std::map; +using std::set; +using std::string; + +void register_test_cluster_watcher() { +} + +class TestClusterWatcher : public ::rbd::mirror::TestFixture { +public: + + TestClusterWatcher() : m_lock("TestClusterWatcherLock") + { + m_cluster = std::make_shared<librados::Rados>(); + EXPECT_EQ("", connect_cluster_pp(*m_cluster)); + } + + ~TestClusterWatcher() override { + m_cluster->wait_for_latest_osdmap(); + for (auto& pool : m_pools) { + EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str())); + } + } + + void SetUp() override { + TestFixture::SetUp(); + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + m_cluster, + m_threads)); + m_cluster_watcher.reset(new ClusterWatcher(m_cluster, m_lock, + m_service_daemon.get())); + } + + void TearDown() override { + m_service_daemon.reset(); + m_cluster_watcher.reset(); + TestFixture::TearDown(); + } + + void create_pool(bool enable_mirroring, const PeerSpec &peer, + string *uuid = nullptr, string *name=nullptr) { + string pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str())); + + int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); + ASSERT_GE(pool_id, 0); + + librados::IoCtx ioctx; + ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); + ioctx.application_enable("rbd", true); + + m_pools.insert(pool_name); + if (enable_mirroring) { + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, + RBD_MIRROR_MODE_POOL)); + + std::string gen_uuid; + ASSERT_EQ(0, librbd::api::Mirror<>::peer_add(ioctx, + uuid != nullptr ? uuid : + &gen_uuid, + peer.cluster_name, + peer.client_name)); + m_pool_peers[pool_id].insert(peer); + } + if (name != nullptr) { + *name = pool_name; + } + } + + void delete_pool(const string &name, const PeerSpec &peer) { + int64_t pool_id = m_cluster->pool_lookup(name.c_str()); + ASSERT_GE(pool_id, 0); + if (m_pool_peers.find(pool_id) != m_pool_peers.end()) { + m_pool_peers[pool_id].erase(peer); + if (m_pool_peers[pool_id].empty()) { + m_pool_peers.erase(pool_id); + } + } + m_pools.erase(name); + ASSERT_EQ(0, m_cluster->pool_delete(name.c_str())); + } + + void set_peer_config_key(const std::string& pool_name, + const PeerSpec &peer) { + int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); + ASSERT_GE(pool_id, 0); + + std::string json = + "{" + "\\\"mon_host\\\": \\\"" + peer.mon_host + "\\\", " + "\\\"key\\\": \\\"" + peer.key + "\\\"" + "}"; + + bufferlist in_bl; + ASSERT_EQ(0, m_cluster->mon_command( + "{" + "\"prefix\": \"config-key set\"," + "\"key\": \"" RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) + + "/" + peer.uuid + "\"," + "\"val\": \"" + json + "\"" + + "}", in_bl, nullptr, nullptr)); + } + + void create_cache_pool(const string &base_pool, string *cache_pool_name) { + bufferlist inbl; + *cache_pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, m_cluster->pool_create(cache_pool_name->c_str())); + + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier add\", \"pool\": \"" + base_pool + + "\", \"tierpool\": \"" + *cache_pool_name + + "\", \"force_nonempty\": \"--force-nonempty\" }", + inbl, NULL, NULL)); + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + base_pool + + "\", \"overlaypool\": \"" + *cache_pool_name + "\"}", + inbl, NULL, NULL)); + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + *cache_pool_name + + "\", \"mode\": \"writeback\"}", + inbl, NULL, NULL)); + m_cluster->wait_for_latest_osdmap(); + } + + void remove_cache_pool(const string &base_pool, const string &cache_pool) { + bufferlist inbl; + // tear down tiers + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + base_pool + + "\"}", + inbl, NULL, NULL)); + ASSERT_EQ(0, m_cluster->mon_command( + "{\"prefix\": \"osd tier remove\", \"pool\": \"" + base_pool + + "\", \"tierpool\": \"" + cache_pool + "\"}", + inbl, NULL, NULL)); + m_cluster->wait_for_latest_osdmap(); + m_cluster->pool_delete(cache_pool.c_str()); + } + + void check_peers() { + m_cluster_watcher->refresh_pools(); + Mutex::Locker l(m_lock); + ASSERT_EQ(m_pool_peers, m_cluster_watcher->get_pool_peers()); + } + + RadosRef m_cluster; + Mutex m_lock; + unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon; + unique_ptr<ClusterWatcher> m_cluster_watcher; + + set<string> m_pools; + ClusterWatcher::PoolPeers m_pool_peers; +}; + +TEST_F(TestClusterWatcher, NoPools) { + check_peers(); +} + +TEST_F(TestClusterWatcher, NoMirroredPools) { + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); +} + +TEST_F(TestClusterWatcher, ReplicatedPools) { + PeerSpec site1("", "site1", "mirror1"); + PeerSpec site2("", "site2", "mirror2"); + string first_pool, last_pool; + check_peers(); + create_pool(true, site1, &site1.uuid, &first_pool); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(false, PeerSpec()); + check_peers(); + create_pool(true, site2, &site2.uuid); + check_peers(); + create_pool(true, site2, &site2.uuid); + check_peers(); + create_pool(true, site2, &site2.uuid, &last_pool); + check_peers(); + delete_pool(first_pool, site1); + check_peers(); + delete_pool(last_pool, site2); + check_peers(); +} + +TEST_F(TestClusterWatcher, CachePools) { + PeerSpec site1("", "site1", "mirror1"); + string base1, base2, cache1, cache2; + create_pool(true, site1, &site1.uuid, &base1); + check_peers(); + + create_cache_pool(base1, &cache1); + BOOST_SCOPE_EXIT( base1, cache1, this_ ) { + this_->remove_cache_pool(base1, cache1); + } BOOST_SCOPE_EXIT_END; + check_peers(); + + create_pool(false, PeerSpec(), nullptr, &base2); + create_cache_pool(base2, &cache2); + BOOST_SCOPE_EXIT( base2, cache2, this_ ) { + this_->remove_cache_pool(base2, cache2); + } BOOST_SCOPE_EXIT_END; + check_peers(); +} + +TEST_F(TestClusterWatcher, ConfigKey) { + REQUIRE(!is_librados_test_stub(*m_cluster)); + + std::string pool_name; + check_peers(); + + PeerSpec site1("", "site1", "mirror1"); + create_pool(true, site1, &site1.uuid, &pool_name); + check_peers(); + + PeerSpec site2("", "site2", "mirror2"); + site2.mon_host = "abc"; + site2.key = "xyz"; + create_pool(false, site2, &site2.uuid); + set_peer_config_key(pool_name, site2); + + check_peers(); +} diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc new file mode 100644 index 00000000..93f311e3 --- /dev/null +++ b/src/test/rbd_mirror/test_ImageDeleter.cc @@ -0,0 +1,301 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2016 SUSE LINUX GmbH + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "include/stringify.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Journal.h" +#include "librbd/internal.h" +#include "librbd/Utils.h" +#include "librbd/api/Image.h" +#include "librbd/api/Mirror.h" +#include "librbd/journal/DisabledPolicy.h" +#include "test/rbd_mirror/test_fixture.h" + +#include "test/librados/test.h" +#include "gtest/gtest.h" + +#define GLOBAL_IMAGE_ID "global_image_id" +#define GLOBAL_CLONE_IMAGE_ID "global_image_id_clone" + +#define dout_subsys ceph_subsys_rbd_mirror + +using rbd::mirror::RadosRef; +using rbd::mirror::TestFixture; +using namespace librbd; +using cls::rbd::MirrorImageState; + + +void register_test_rbd_mirror_image_deleter() { +} + +class TestImageDeleter : public TestFixture { +public: + const std::string m_local_mirror_uuid = "local mirror uuid"; + const std::string m_remote_mirror_uuid = "remote mirror uuid"; + + void SetUp() override { + TestFixture::SetUp(); + + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + _rados, m_threads)); + + librbd::api::Mirror<>::mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE); + + m_deleter = new rbd::mirror::ImageDeleter<>(m_local_io_ctx, m_threads, + m_service_daemon.get()); + + m_local_image_id = librbd::util::generate_image_id(m_local_io_ctx); + librbd::ImageOptions image_opts; + image_opts.set(RBD_IMAGE_OPTION_FEATURES, RBD_FEATURES_ALL); + EXPECT_EQ(0, librbd::create(m_local_io_ctx, m_image_name, m_local_image_id, + 1 << 20, image_opts, GLOBAL_IMAGE_ID, + m_remote_mirror_uuid, true)); + + cls::rbd::MirrorImage mirror_image( + GLOBAL_IMAGE_ID, MirrorImageState::MIRROR_IMAGE_STATE_ENABLED); + EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, m_local_image_id, + mirror_image)); + } + + void TearDown() override { + remove_image(); + + C_SaferCond ctx; + m_deleter->shut_down(&ctx); + ctx.wait(); + + delete m_deleter; + m_service_daemon.reset(); + + TestFixture::TearDown(); + } + + void init_image_deleter() { + C_SaferCond ctx; + m_deleter->init(&ctx); + ASSERT_EQ(0, ctx.wait()); + } + + void remove_image() { + cls::rbd::MirrorImage mirror_image; + int r = cls_client::mirror_image_get(&m_local_io_ctx, m_local_image_id, + &mirror_image); + EXPECT_EQ(1, r == 0 || r == -ENOENT); + if (r != -ENOENT) { + mirror_image.state = MirrorImageState::MIRROR_IMAGE_STATE_ENABLED; + EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, + m_local_image_id, + mirror_image)); + } + promote_image(); + + NoOpProgressContext ctx; + r = librbd::api::Image<>::remove(m_local_io_ctx, m_image_name, ctx); + EXPECT_EQ(1, r == 0 || r == -ENOENT); + } + + void promote_image(ImageCtx *ictx=nullptr) { + bool close = false; + int r = 0; + if (!ictx) { + ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + r = ictx->state->open(0); + close = (r == 0); + } + + EXPECT_EQ(1, r == 0 || r == -ENOENT); + + if (r == 0) { + int r2 = librbd::api::Mirror<>::image_promote(ictx, true); + EXPECT_EQ(1, r2 == 0 || r2 == -EINVAL); + } + + if (close) { + EXPECT_EQ(0, ictx->state->close()); + } + } + + void demote_image(ImageCtx *ictx=nullptr) { + bool close = false; + if (!ictx) { + ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(0, ictx->state->open(0)); + close = true; + } + + EXPECT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); + + if (close) { + EXPECT_EQ(0, ictx->state->close()); + } + } + + void create_snapshot(std::string snap_name="snap1", bool protect=false) { + ImageCtx *ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(0, ictx->state->open(0)); + { + RWLock::WLocker snap_locker(ictx->snap_lock); + ictx->set_journal_policy(new librbd::journal::DisabledPolicy()); + } + + EXPECT_EQ(0, ictx->operations->snap_create( + cls::rbd::UserSnapshotNamespace(), snap_name.c_str())); + + if (protect) { + EXPECT_EQ(0, ictx->operations->snap_protect( + cls::rbd::UserSnapshotNamespace(), snap_name.c_str())); + } + + EXPECT_EQ(0, ictx->state->close()); + } + + std::string create_clone() { + ImageCtx *ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(0, ictx->state->open(0)); + { + RWLock::WLocker snap_locker(ictx->snap_lock); + ictx->set_journal_policy(new librbd::journal::DisabledPolicy()); + } + + EXPECT_EQ(0, ictx->operations->snap_create( + cls::rbd::UserSnapshotNamespace(), "snap1")); + EXPECT_EQ(0, ictx->operations->snap_protect( + cls::rbd::UserSnapshotNamespace(), "snap1")); + EXPECT_EQ(0, librbd::api::Image<>::snap_set( + ictx, cls::rbd::UserSnapshotNamespace(), "snap1")); + + std::string clone_id = librbd::util::generate_image_id(m_local_io_ctx); + librbd::ImageOptions clone_opts; + clone_opts.set(RBD_IMAGE_OPTION_FEATURES, ictx->features); + EXPECT_EQ(0, librbd::clone(m_local_io_ctx, m_local_image_id.c_str(), + nullptr, "snap1", m_local_io_ctx, + clone_id.c_str(), "clone1", clone_opts, + GLOBAL_CLONE_IMAGE_ID, m_remote_mirror_uuid)); + + cls::rbd::MirrorImage mirror_image( + GLOBAL_CLONE_IMAGE_ID, MirrorImageState::MIRROR_IMAGE_STATE_ENABLED); + EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, clone_id, + mirror_image)); + EXPECT_EQ(0, ictx->state->close()); + return clone_id; + } + + void check_image_deleted() { + ImageCtx *ictx = new ImageCtx("", m_local_image_id, "", m_local_io_ctx, + false); + EXPECT_EQ(-ENOENT, ictx->state->open(0)); + + cls::rbd::MirrorImage mirror_image; + EXPECT_EQ(-ENOENT, cls_client::mirror_image_get(&m_local_io_ctx, + m_local_image_id, + &mirror_image)); + } + + int trash_move(const std::string& global_image_id) { + C_SaferCond ctx; + rbd::mirror::ImageDeleter<>::trash_move(m_local_io_ctx, global_image_id, + true, m_threads->work_queue, &ctx); + return ctx.wait(); + } + + librbd::RBD rbd; + std::string m_local_image_id; + std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon; + rbd::mirror::ImageDeleter<> *m_deleter; +}; + +TEST_F(TestImageDeleter, ExistingTrashMove) { + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + init_image_deleter(); + + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestImageDeleter, LiveTrashMove) { + init_image_deleter(); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestImageDeleter, Delete_Image_With_Snapshots) { + init_image_deleter(); + create_snapshot("snap1"); + create_snapshot("snap2"); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + EXPECT_EQ(0, ctx.wait()); + + ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); + ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); +} + +TEST_F(TestImageDeleter, Delete_Image_With_ProtectedSnapshots) { + init_image_deleter(); + create_snapshot("snap1", true); + create_snapshot("snap2", true); + + C_SaferCond ctx; + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx); + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + EXPECT_EQ(0, ctx.wait()); + + ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); + ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); +} + +TEST_F(TestImageDeleter, Delete_Image_With_Clone) { + init_image_deleter(); + std::string clone_id = create_clone(); + + C_SaferCond ctx1; + m_deleter->set_busy_timer_interval(0.1); + m_deleter->wait_for_deletion(m_local_image_id, false, &ctx1); + ASSERT_EQ(0, trash_move(GLOBAL_IMAGE_ID)); + EXPECT_EQ(-EBUSY, ctx1.wait()); + + C_SaferCond ctx2; + m_deleter->wait_for_deletion(clone_id, false, &ctx2); + ASSERT_EQ(0, trash_move(GLOBAL_CLONE_IMAGE_ID)); + EXPECT_EQ(0, ctx2.wait()); + + C_SaferCond ctx3; + m_deleter->wait_for_deletion(m_local_image_id, true, &ctx3); + EXPECT_EQ(0, ctx3.wait()); + + ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); + ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); +} + diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc new file mode 100644 index 00000000..67adc18a --- /dev/null +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -0,0 +1,1162 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph distributed storage system + * + * Copyright (C) 2016 Mirantis Inc + * + * Author: Mykola Golub <mgolub@mirantis.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + */ + +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "include/stringify.h" +#include "test/rbd_mirror/test_fixture.h" +#include "cls/journal/cls_journal_types.h" +#include "cls/journal/cls_journal_client.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "journal/Journaler.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/internal.h" +#include "librbd/api/Mirror.h" +#include "librbd/io/AioCompletion.h" +#include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ReadResult.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" + +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" + +using rbd::mirror::RadosRef; + +void register_test_rbd_mirror() { +} + +#define TEST_IO_SIZE 512 +#define TEST_IO_COUNT 11 + +class TestImageReplayer : public ::rbd::mirror::TestFixture { +public: + struct C_WatchCtx : public librados::WatchCtx2 { + TestImageReplayer *test; + std::string oid; + Mutex lock; + Cond cond; + bool notified; + + C_WatchCtx(TestImageReplayer *test, const std::string &oid) + : test(test), oid(oid), lock("C_WatchCtx::lock"), notified(false) { + } + + void handle_notify(uint64_t notify_id, uint64_t cookie, + uint64_t notifier_id, bufferlist& bl_) override { + bufferlist bl; + test->m_remote_ioctx.notify_ack(oid, notify_id, cookie, bl); + + Mutex::Locker locker(lock); + notified = true; + cond.Signal(); + } + + void handle_error(uint64_t cookie, int err) override { + ASSERT_EQ(0, err); + } + }; + + TestImageReplayer() + : m_local_cluster(new librados::Rados()), m_watch_handle(0) + { + EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get())); + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false")); + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1")); + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_commit_age", + "0.1")); + m_local_pool_name = get_temp_pool_name(); + EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str())); + EXPECT_EQ(0, m_local_cluster->ioctx_create(m_local_pool_name.c_str(), + m_local_ioctx)); + m_local_ioctx.application_enable("rbd", true); + + EXPECT_EQ("", connect_cluster_pp(m_remote_cluster)); + EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false")); + + m_remote_pool_name = get_temp_pool_name(); + EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str())); + m_remote_pool_id = m_remote_cluster.pool_lookup(m_remote_pool_name.c_str()); + EXPECT_GE(m_remote_pool_id, 0); + + EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(), + m_remote_ioctx)); + m_remote_ioctx.application_enable("rbd", true); + + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_POOL)); + + m_image_name = get_temp_image_name(); + uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); + features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; + int order = 0; + EXPECT_EQ(0, librbd::create(m_remote_ioctx, m_image_name.c_str(), 1 << 22, + false, features, &order, 0, 0)); + m_remote_image_id = get_image_id(m_remote_ioctx, m_image_name); + m_global_image_id = get_global_image_id(m_remote_ioctx, m_remote_image_id); + + m_threads.reset(new rbd::mirror::Threads<>(reinterpret_cast<CephContext*>( + m_local_ioctx.cct()))); + + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, + m_local_cluster, + m_threads.get())); + + m_instance_watcher = rbd::mirror::InstanceWatcher<>::create( + m_local_ioctx, m_threads->work_queue, nullptr); + m_instance_watcher->handle_acquire_leader(); + } + + ~TestImageReplayer() override + { + unwatch(); + + m_instance_watcher->handle_release_leader(); + + delete m_replayer; + delete m_instance_watcher; + + EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str())); + EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str())); + } + + template <typename ImageReplayerT = rbd::mirror::ImageReplayer<> > + void create_replayer() { + m_replayer = new ImageReplayerT( + m_threads.get(), m_instance_watcher, + rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)), + m_local_mirror_uuid, m_local_ioctx.get_id(), m_global_image_id); + m_replayer->add_peer("peer uuid", m_remote_ioctx); + } + + void start() + { + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + + ASSERT_EQ(0U, m_watch_handle); + std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); + m_watch_ctx = new C_WatchCtx(this, oid); + ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx)); + } + + void unwatch() { + if (m_watch_handle != 0) { + m_remote_ioctx.unwatch2(m_watch_handle); + delete m_watch_ctx; + m_watch_ctx = nullptr; + m_watch_handle = 0; + } + } + + void stop() + { + unwatch(); + + C_SaferCond cond; + m_replayer->stop(&cond); + ASSERT_EQ(0, cond.wait()); + } + + void bootstrap() + { + create_replayer<>(); + + start(); + wait_for_replay_complete(); + stop(); + } + + std::string get_temp_image_name() + { + return "image" + stringify(++_image_number); + } + + std::string get_image_id(librados::IoCtx &ioctx, const string &image_name) + { + std::string obj = librbd::util::id_obj_name(image_name); + std::string id; + EXPECT_EQ(0, librbd::cls_client::get_id(&ioctx, obj, &id)); + return id; + } + + std::string get_global_image_id(librados::IoCtx& io_ctx, + const std::string& image_id) { + cls::rbd::MirrorImage mirror_image; + EXPECT_EQ(0, librbd::cls_client::mirror_image_get(&io_ctx, image_id, + &mirror_image)); + return mirror_image.global_image_id; + } + + void open_image(librados::IoCtx &ioctx, const std::string &image_name, + bool readonly, librbd::ImageCtx **ictxp) + { + librbd::ImageCtx *ictx = new librbd::ImageCtx(image_name.c_str(), + "", "", ioctx, readonly); + EXPECT_EQ(0, ictx->state->open(0)); + *ictxp = ictx; + } + + void open_local_image(librbd::ImageCtx **ictxp) + { + open_image(m_local_ioctx, m_image_name, true, ictxp); + } + + void open_remote_image(librbd::ImageCtx **ictxp) + { + open_image(m_remote_ioctx, m_image_name, false, ictxp); + } + + void close_image(librbd::ImageCtx *ictx) + { + ictx->state->close(); + } + + void get_commit_positions(cls::journal::ObjectPosition *master_position, + cls::journal::ObjectPosition *mirror_position) + { + std::string master_client_id = ""; + std::string mirror_client_id = m_local_mirror_uuid; + + C_SaferCond cond; + uint64_t minimum_set; + uint64_t active_set; + std::set<cls::journal::Client> registered_clients; + std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); + cls::journal::client::get_mutable_metadata(m_remote_ioctx, oid, + &minimum_set, &active_set, + ®istered_clients, &cond); + ASSERT_EQ(0, cond.wait()); + + *master_position = cls::journal::ObjectPosition(); + *mirror_position = cls::journal::ObjectPosition(); + + std::set<cls::journal::Client>::const_iterator c; + for (c = registered_clients.begin(); c != registered_clients.end(); ++c) { + std::cout << __func__ << ": client: " << *c << std::endl; + if (c->state != cls::journal::CLIENT_STATE_CONNECTED) { + continue; + } + cls::journal::ObjectPositions object_positions = + c->commit_position.object_positions; + cls::journal::ObjectPositions::const_iterator p = + object_positions.begin(); + if (p != object_positions.end()) { + if (c->id == master_client_id) { + ASSERT_EQ(cls::journal::ObjectPosition(), *master_position); + *master_position = *p; + } else if (c->id == mirror_client_id) { + ASSERT_EQ(cls::journal::ObjectPosition(), *mirror_position); + *mirror_position = *p; + } + } + } + } + + bool wait_for_watcher_notify(int seconds) + { + if (m_watch_handle == 0) { + return false; + } + + Mutex::Locker locker(m_watch_ctx->lock); + while (!m_watch_ctx->notified) { + if (m_watch_ctx->cond.WaitInterval(m_watch_ctx->lock, + utime_t(seconds, 0)) != 0) { + return false; + } + } + m_watch_ctx->notified = false; + return true; + } + + void wait_for_replay_complete() + { + cls::journal::ObjectPosition master_position; + cls::journal::ObjectPosition mirror_position; + + for (int i = 0; i < 100; i++) { + get_commit_positions(&master_position, &mirror_position); + if (master_position == mirror_position) { + break; + } + wait_for_watcher_notify(1); + } + + ASSERT_EQ(master_position, mirror_position); + } + + void wait_for_stopped() { + for (int i = 0; i < 100; i++) { + if (m_replayer->is_stopped()) { + break; + } + wait_for_watcher_notify(1); + } + ASSERT_TRUE(m_replayer->is_stopped()); + } + + void write_test_data(librbd::ImageCtx *ictx, const char *test_data, off_t off, + size_t len) + { + size_t written; + bufferlist bl; + bl.append(std::string(test_data, len)); + written = ictx->io_work_queue->write(off, len, std::move(bl), 0); + printf("wrote: %d\n", (int)written); + ASSERT_EQ(len, written); + } + + void read_test_data(librbd::ImageCtx *ictx, const char *expected, off_t off, + size_t len) + { + ssize_t read; + char *result = (char *)malloc(len + 1); + + ASSERT_NE(static_cast<char *>(NULL), result); + read = ictx->io_work_queue->read( + off, len, librbd::io::ReadResult{result, len}, 0); + printf("read: %d\n", (int)read); + ASSERT_EQ(len, static_cast<size_t>(read)); + result[len] = '\0'; + if (memcmp(result, expected, len)) { + printf("read: %s\nexpected: %s\n", result, expected); + ASSERT_EQ(0, memcmp(result, expected, len)); + } + free(result); + } + + void generate_test_data() { + for (int i = 0; i < TEST_IO_SIZE; ++i) { + m_test_data[i] = (char) (rand() % (126 - 33) + 33); + } + m_test_data[TEST_IO_SIZE] = '\0'; + } + + void flush(librbd::ImageCtx *ictx) + { + C_SaferCond aio_flush_ctx; + auto c = librbd::io::AioCompletion::create(&aio_flush_ctx); + c->get(); + ictx->io_work_queue->aio_flush(c); + ASSERT_EQ(0, c->wait_for_complete()); + c->put(); + + C_SaferCond journal_flush_ctx; + ictx->journal->flush_commit_position(&journal_flush_ctx); + ASSERT_EQ(0, journal_flush_ctx.wait()); + + printf("flushed\n"); + } + + static int _image_number; + + std::shared_ptr<librados::Rados> m_local_cluster; + std::unique_ptr<rbd::mirror::Threads<>> m_threads; + std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon; + librados::Rados m_remote_cluster; + rbd::mirror::InstanceWatcher<> *m_instance_watcher; + std::string m_local_mirror_uuid = "local mirror uuid"; + std::string m_remote_mirror_uuid = "remote mirror uuid"; + std::string m_local_pool_name, m_remote_pool_name; + librados::IoCtx m_local_ioctx, m_remote_ioctx; + std::string m_image_name; + int64_t m_remote_pool_id; + std::string m_remote_image_id; + std::string m_global_image_id; + rbd::mirror::ImageReplayer<> *m_replayer; + C_WatchCtx *m_watch_ctx; + uint64_t m_watch_handle; + char m_test_data[TEST_IO_SIZE + 1]; + std::string m_journal_commit_age; +}; + +int TestImageReplayer::_image_number; + +TEST_F(TestImageReplayer, Bootstrap) +{ + bootstrap(); +} + +TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists) +{ + int order = 0; + EXPECT_EQ(0, librbd::create(m_local_ioctx, m_image_name.c_str(), 1 << 22, + false, 0, &order, 0, 0)); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(-EEXIST, cond.wait()); +} + +TEST_F(TestImageReplayer, BootstrapErrorNoJournal) +{ + ASSERT_EQ(0, librbd::Journal<>::remove(m_remote_ioctx, m_remote_image_id)); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(-ENOENT, cond.wait()); +} + +TEST_F(TestImageReplayer, BootstrapErrorMirrorDisabled) +{ + // disable remote image mirroring + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_IMAGE)); + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::api::Mirror<>::image_disable(ictx, true)); + close_image(ictx); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(-ENOENT, cond.wait()); +} + +TEST_F(TestImageReplayer, BootstrapMirrorDisabling) +{ + // set remote image mirroring state to DISABLING + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx, + RBD_MIRROR_MODE_IMAGE)); + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::api::Mirror<>::image_enable(ictx, false)); + cls::rbd::MirrorImage mirror_image; + ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&m_remote_ioctx, ictx->id, + &mirror_image)); + mirror_image.state = cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING; + ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&m_remote_ioctx, ictx->id, + mirror_image)); + close_image(ictx); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(-EREMOTEIO, cond.wait()); + ASSERT_TRUE(m_replayer->is_stopped()); +} + +TEST_F(TestImageReplayer, BootstrapDemoted) +{ + // demote remote image + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::api::Mirror<>::image_demote(ictx)); + close_image(ictx); + + create_replayer<>(); + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(-EREMOTEIO, cond.wait()); + ASSERT_TRUE(m_replayer->is_stopped()); +} + +TEST_F(TestImageReplayer, StartInterrupted) +{ + create_replayer<>(); + C_SaferCond start_cond, stop_cond; + m_replayer->start(&start_cond); + m_replayer->stop(&stop_cond); + int r = start_cond.wait(); + printf("start returned %d\n", r); + // TODO: improve the test to avoid this race + ASSERT_TRUE(r == -ECANCELED || r == 0); + ASSERT_EQ(0, stop_cond.wait()); +} + +TEST_F(TestImageReplayer, JournalReset) +{ + bootstrap(); + delete m_replayer; + + ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id)); + + // try to recover + bootstrap(); +} + +TEST_F(TestImageReplayer, ErrorNoJournal) +{ + bootstrap(); + + // disable remote journal journaling + // (reset before disabling, so it does not fail with EBUSY) + ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id)); + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + uint64_t features; + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0, ictx->operations->update_features(RBD_FEATURE_JOURNALING, + false)); + close_image(ictx); + + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); +} + +TEST_F(TestImageReplayer, StartStop) +{ + bootstrap(); + + start(); + wait_for_replay_complete(); + stop(); +} + +TEST_F(TestImageReplayer, WriteAndStartReplay) +{ + bootstrap(); + + // Write to remote image and start replay + + librbd::ImageCtx *ictx; + + generate_test_data(); + open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + start(); + wait_for_replay_complete(); + stop(); + + open_local_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); +} + +TEST_F(TestImageReplayer, StartReplayAndWrite) +{ + bootstrap(); + + // Start replay and write to remote image + + librbd::ImageCtx *ictx; + + start(); + + generate_test_data(); + open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + + wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + wait_for_replay_complete(); + + open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + + stop(); +} + +TEST_F(TestImageReplayer, NextTag) +{ + bootstrap(); + + // write, reopen, and write again to test switch to the next tag + + librbd::ImageCtx *ictx; + + start(); + + generate_test_data(); + + const int N = 10; + + for (int j = 0; j < N; j++) { + open_remote_image(&ictx); + for (int i = j * TEST_IO_COUNT; i < (j + 1) * TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + } + + wait_for_replay_complete(); + + open_local_image(&ictx); + for (int i = 0; i < N * TEST_IO_COUNT; ++i) { + read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + + stop(); +} + +TEST_F(TestImageReplayer, Resync) +{ + bootstrap(); + + librbd::ImageCtx *ictx; + + start(); + + generate_test_data(); + + open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + + wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + C_SaferCond ctx; + m_replayer->resync_image(&ctx); + ASSERT_EQ(0, ctx.wait()); + + wait_for_stopped(); + + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + + ASSERT_TRUE(m_replayer->is_replaying()); + wait_for_replay_complete(); + + open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + + stop(); +} + +TEST_F(TestImageReplayer, Resync_While_Stop) +{ + + bootstrap(); + + start(); + + generate_test_data(); + + librbd::ImageCtx *ictx; + open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + + wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + wait_for_replay_complete(); + + C_SaferCond cond; + m_replayer->stop(&cond); + ASSERT_EQ(0, cond.wait()); + + open_local_image(&ictx); + librbd::Journal<>::request_resync(ictx); + close_image(ictx); + + C_SaferCond cond2; + m_replayer->start(&cond2); + ASSERT_EQ(0, cond2.wait()); + + ASSERT_TRUE(m_replayer->is_stopped()); + + C_SaferCond cond3; + m_replayer->start(&cond3); + ASSERT_EQ(0, cond3.wait()); + + ASSERT_TRUE(m_replayer->is_replaying()); + + wait_for_replay_complete(); + + open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + + stop(); +} + +TEST_F(TestImageReplayer, Resync_StartInterrupted) +{ + + bootstrap(); + + librbd::ImageCtx *ictx; + open_local_image(&ictx); + librbd::Journal<>::request_resync(ictx); + close_image(ictx); + + C_SaferCond cond; + m_replayer->start(&cond); + ASSERT_EQ(0, cond.wait()); + + ASSERT_TRUE(m_replayer->is_stopped()); + + C_SaferCond cond2; + m_replayer->start(&cond2); + ASSERT_EQ(0, cond2.wait()); + + ASSERT_EQ(0U, m_watch_handle); + std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); + m_watch_ctx = new C_WatchCtx(this, oid); + ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx)); + + ASSERT_TRUE(m_replayer->is_replaying()); + + generate_test_data(); + open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + + wait_for_replay_complete(); + + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + wait_for_replay_complete(); + + open_local_image(&ictx); + for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) { + read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + + stop(); +} + +TEST_F(TestImageReplayer, MultipleReplayFailures_SingleEpoch) { + bootstrap(); + + // inject a snapshot that cannot be unprotected + librbd::ImageCtx *ictx; + open_image(m_local_ioctx, m_image_name, false, &ictx); + ictx->features &= ~RBD_FEATURE_JOURNALING; + ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + "foo")); + ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + "foo")); + ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, + {ictx->md_ctx.get_id(), "", + ictx->id, + ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), "foo"}]}, + "dummy child id")); + close_image(ictx); + + // race failed op shut down with new ops + open_remote_image(&ictx); + for (uint64_t i = 0; i < 10; ++i) { + RWLock::RLocker owner_locker(ictx->owner_lock); + C_SaferCond request_lock; + ictx->exclusive_lock->acquire_lock(&request_lock); + ASSERT_EQ(0, request_lock.wait()); + + C_SaferCond append_ctx; + ictx->journal->append_op_event( + i, + librbd::journal::EventEntry{ + librbd::journal::SnapUnprotectEvent{i, + cls::rbd::UserSnapshotNamespace(), + "foo"}}, + &append_ctx); + ASSERT_EQ(0, append_ctx.wait()); + + C_SaferCond commit_ctx; + ictx->journal->commit_op_event(i, 0, &commit_ctx); + ASSERT_EQ(0, commit_ctx.wait()); + + C_SaferCond release_ctx; + ictx->exclusive_lock->release_lock(&release_ctx); + ASSERT_EQ(0, release_ctx.wait()); + } + + for (uint64_t i = 0; i < 5; ++i) { + start(); + wait_for_stopped(); + unwatch(); + } + close_image(ictx); +} + +TEST_F(TestImageReplayer, MultipleReplayFailures_MultiEpoch) { + bootstrap(); + + // inject a snapshot that cannot be unprotected + librbd::ImageCtx *ictx; + open_image(m_local_ioctx, m_image_name, false, &ictx); + ictx->features &= ~RBD_FEATURE_JOURNALING; + ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + "foo")); + ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + "foo")); + ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN, + {ictx->md_ctx.get_id(), "", + ictx->id, + ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), + "foo"}]}, + "dummy child id")); + close_image(ictx); + + // race failed op shut down with new tag flush + open_remote_image(&ictx); + { + RWLock::RLocker owner_locker(ictx->owner_lock); + C_SaferCond request_lock; + ictx->exclusive_lock->acquire_lock(&request_lock); + ASSERT_EQ(0, request_lock.wait()); + + C_SaferCond append_ctx; + ictx->journal->append_op_event( + 1U, + librbd::journal::EventEntry{ + librbd::journal::SnapUnprotectEvent{1U, + cls::rbd::UserSnapshotNamespace(), + "foo"}}, + &append_ctx); + ASSERT_EQ(0, append_ctx.wait()); + + C_SaferCond commit_ctx; + ictx->journal->commit_op_event(1U, 0, &commit_ctx); + ASSERT_EQ(0, commit_ctx.wait()); + + C_SaferCond release_ctx; + ictx->exclusive_lock->release_lock(&release_ctx); + ASSERT_EQ(0, release_ctx.wait()); + } + + generate_test_data(); + write_test_data(ictx, m_test_data, 0, TEST_IO_SIZE); + + for (uint64_t i = 0; i < 5; ++i) { + start(); + wait_for_stopped(); + unwatch(); + } + close_image(ictx); +} + +TEST_F(TestImageReplayer, Disconnect) +{ + bootstrap(); + + // Make sure rbd_mirroring_resync_after_disconnect is not set + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "false")); + + // Test start fails if disconnected + + librbd::ImageCtx *ictx; + + generate_test_data(); + open_remote_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + std::string oid = ::journal::Journaler::header_oid(m_remote_image_id); + ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid, + m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED)); + + C_SaferCond cond1; + m_replayer->start(&cond1); + ASSERT_EQ(-ENOTCONN, cond1.wait()); + + // Test start succeeds after resync + + open_local_image(&ictx); + librbd::Journal<>::request_resync(ictx); + close_image(ictx); + C_SaferCond cond2; + m_replayer->start(&cond2); + ASSERT_EQ(0, cond2.wait()); + + start(); + wait_for_replay_complete(); + + // Test replay stopped after disconnect + + open_remote_image(&ictx); + for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid, + m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED)); + bufferlist bl; + ASSERT_EQ(0, m_remote_ioctx.notify2(oid, bl, 5000, NULL)); + + wait_for_stopped(); + + // Test start fails after disconnect + + C_SaferCond cond3; + m_replayer->start(&cond3); + ASSERT_EQ(-ENOTCONN, cond3.wait()); + C_SaferCond cond4; + m_replayer->start(&cond4); + ASSERT_EQ(-ENOTCONN, cond4.wait()); + + // Test automatic resync if rbd_mirroring_resync_after_disconnect is set + + EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "true")); + + // Resync is flagged on first start attempt + C_SaferCond cond5; + m_replayer->start(&cond5); + ASSERT_EQ(-ENOTCONN, cond5.wait()); + + C_SaferCond cond6; + m_replayer->start(&cond6); + ASSERT_EQ(0, cond6.wait()); + wait_for_replay_complete(); + + stop(); +} + +TEST_F(TestImageReplayer, UpdateFeatures) +{ + const uint64_t FEATURES_TO_UPDATE = + RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF; + + uint64_t features; + librbd::ImageCtx *ictx; + + // Make sure the features we will update are disabled initially + + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + features &= FEATURES_TO_UPDATE; + if (features) { + ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, + false)); + } + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + close_image(ictx); + + bootstrap(); + + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + close_image(ictx); + + open_local_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + close_image(ictx); + + // Start replay and update features + + start(); + + open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, + true)); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); + close_image(ictx); + + wait_for_replay_complete(); + + open_local_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE); + close_image(ictx); + + open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE, + false)); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + close_image(ictx); + + wait_for_replay_complete(); + + open_local_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_EQ(0U, features & FEATURES_TO_UPDATE); + close_image(ictx); + + // Test update_features error does not stop replication + + open_remote_image(&ictx); + ASSERT_EQ(0, librbd::get_features(ictx, &features)); + ASSERT_NE(0U, features & RBD_FEATURE_EXCLUSIVE_LOCK); + ASSERT_EQ(-EINVAL, ictx->operations->update_features(RBD_FEATURE_EXCLUSIVE_LOCK, + false)); + generate_test_data(); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + wait_for_replay_complete(); + + open_local_image(&ictx); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + + stop(); +} + +TEST_F(TestImageReplayer, MetadataSetRemove) +{ + const std::string KEY = "test_key"; + const std::string VALUE = "test_value"; + + librbd::ImageCtx *ictx; + std::string value; + + bootstrap(); + + start(); + + // Test metadata_set replication + + open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->metadata_set(KEY, VALUE)); + value.clear(); + ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); + ASSERT_EQ(VALUE, value); + close_image(ictx); + + wait_for_replay_complete(); + + open_local_image(&ictx); + value.clear(); + ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value)); + ASSERT_EQ(VALUE, value); + close_image(ictx); + + // Test metadata_remove replication + + open_remote_image(&ictx); + ASSERT_EQ(0, ictx->operations->metadata_remove(KEY)); + ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); + close_image(ictx); + + wait_for_replay_complete(); + + open_local_image(&ictx); + ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value)); + close_image(ictx); + + stop(); +} + +TEST_F(TestImageReplayer, MirroringDelay) +{ + const double DELAY = 10; // set less than wait_for_replay_complete timeout + + librbd::ImageCtx *ictx; + utime_t start_time; + double delay; + + bootstrap(); + + ASSERT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_replay_delay", + stringify(DELAY).c_str())); + open_local_image(&ictx); + ASSERT_EQ(DELAY, ictx->mirroring_replay_delay); + close_image(ictx); + + start(); + + // Test delay + + generate_test_data(); + open_remote_image(&ictx); + start_time = ceph_clock_now(); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + flush(ictx); + close_image(ictx); + + wait_for_replay_complete(); + delay = ceph_clock_now() - start_time; + ASSERT_GE(delay, DELAY); + + // Test stop when delaying replay + + open_remote_image(&ictx); + start_time = ceph_clock_now(); + for (int i = 0; i < TEST_IO_COUNT; ++i) { + write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE); + } + close_image(ictx); + + sleep(DELAY / 2); + stop(); + start(); + + wait_for_replay_complete(); + delay = ceph_clock_now() - start_time; + ASSERT_GE(delay, DELAY); + + stop(); +} diff --git a/src/test/rbd_mirror/test_ImageSync.cc b/src/test/rbd_mirror/test_ImageSync.cc new file mode 100644 index 00000000..7f9ae105 --- /dev/null +++ b/src/test/rbd_mirror/test_ImageSync.cc @@ -0,0 +1,347 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_fixture.h" +#include "include/stringify.h" +#include "include/rbd/librbd.hpp" +#include "journal/Journaler.h" +#include "journal/Settings.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/internal.h" +#include "librbd/Operations.h" +#include "librbd/io/AioCompletion.h" +#include "librbd/io/ImageDispatchSpec.h" +#include "librbd/io/ImageRequestWQ.h" +#include "librbd/io/ReadResult.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/ImageSync.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" + +void register_test_image_sync() { +} + +namespace rbd { +namespace mirror { + +namespace { + +int flush(librbd::ImageCtx *image_ctx) { + C_SaferCond ctx; + auto aio_comp = librbd::io::AioCompletion::create_and_start( + &ctx, image_ctx, librbd::io::AIO_TYPE_FLUSH); + auto req = librbd::io::ImageDispatchSpec<>::create_flush_request( + *image_ctx, aio_comp, librbd::io::FLUSH_SOURCE_INTERNAL, {}); + req->send(); + delete req; + return ctx.wait(); +} + +void scribble(librbd::ImageCtx *image_ctx, int num_ops, uint64_t max_size) +{ + max_size = std::min<uint64_t>(image_ctx->size, max_size); + for (int i=0; i<num_ops; i++) { + uint64_t off = rand() % (image_ctx->size - max_size + 1); + uint64_t len = 1 + rand() % max_size; + + if (rand() % 4 == 0) { + ASSERT_EQ((int)len, + image_ctx->io_work_queue->discard( + off, len, image_ctx->discard_granularity_bytes)); + } else { + bufferlist bl; + bl.append(std::string(len, '1')); + ASSERT_EQ((int)len, image_ctx->io_work_queue->write(off, len, + std::move(bl), 0)); + } + } + + RWLock::RLocker owner_locker(image_ctx->owner_lock); + ASSERT_EQ(0, flush(image_ctx)); +} + +} // anonymous namespace +class TestImageSync : public TestFixture { +public: + + void SetUp() override { + TestFixture::SetUp(); + create_and_open(m_local_io_ctx, &m_local_image_ctx); + create_and_open(m_remote_io_ctx, &m_remote_image_ctx); + + m_instance_watcher = rbd::mirror::InstanceWatcher<>::create( + m_local_io_ctx, m_threads->work_queue, nullptr); + m_instance_watcher->handle_acquire_leader(); + + m_remote_journaler = new ::journal::Journaler( + m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, + m_remote_io_ctx, m_remote_image_ctx->id, "mirror-uuid", {}); + + m_client_meta = {"image-id"}; + + librbd::journal::ClientData client_data(m_client_meta); + bufferlist client_data_bl; + encode(client_data, client_data_bl); + + ASSERT_EQ(0, m_remote_journaler->register_client(client_data_bl)); + } + + void TearDown() override { + TestFixture::TearDown(); + + m_instance_watcher->handle_release_leader(); + + delete m_remote_journaler; + delete m_instance_watcher; + } + + void create_and_open(librados::IoCtx &io_ctx, librbd::ImageCtx **image_ctx) { + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(io_ctx, m_image_name, image_ctx)); + + C_SaferCond ctx; + { + RWLock::RLocker owner_locker((*image_ctx)->owner_lock); + (*image_ctx)->exclusive_lock->try_acquire_lock(&ctx); + } + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE((*image_ctx)->exclusive_lock->is_lock_owner()); + } + + ImageSync<> *create_request(Context *ctx) { + return new ImageSync<>(m_local_image_ctx, m_remote_image_ctx, + m_threads->timer, &m_threads->timer_lock, + "mirror-uuid", m_remote_journaler, &m_client_meta, + m_threads->work_queue, m_instance_watcher, ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx; + rbd::mirror::InstanceWatcher<> *m_instance_watcher; + ::journal::Journaler *m_remote_journaler; + librbd::journal::MirrorPeerClientMeta m_client_meta; +}; + +TEST_F(TestImageSync, Empty) { + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_EQ(0U, m_client_meta.sync_points.size()); + ASSERT_EQ(0, m_remote_image_ctx->state->refresh()); + ASSERT_EQ(0U, m_remote_image_ctx->snap_ids.size()); + ASSERT_EQ(0, m_local_image_ctx->state->refresh()); + ASSERT_EQ(1U, m_local_image_ctx->snap_ids.size()); // deleted on journal replay +} + +TEST_F(TestImageSync, Simple) { + scribble(m_remote_image_ctx, 10, 102400); + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + bufferlist read_remote_bl; + read_remote_bl.append(std::string(object_size, '1')); + bufferlist read_local_bl; + read_local_bl.append(std::string(object_size, '1')); + + for (uint64_t offset = 0; offset < m_remote_image_ctx->size; + offset += object_size) { + ASSERT_LE(0, m_remote_image_ctx->io_work_queue->read( + offset, object_size, + librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, m_local_image_ctx->io_work_queue->read( + offset, object_size, + librbd::io::ReadResult{&read_local_bl}, 0)); + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); + } +} + +TEST_F(TestImageSync, Resize) { + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + + uint64_t off = 0; + uint64_t len = object_size / 10; + + bufferlist bl; + bl.append(std::string(len, '1')); + ASSERT_EQ((int)len, m_remote_image_ctx->io_work_queue->write(off, len, + std::move(bl), + 0)); + { + RWLock::RLocker owner_locker(m_remote_image_ctx->owner_lock); + ASSERT_EQ(0, flush(m_remote_image_ctx)); + } + + ASSERT_EQ(0, create_snap(m_remote_image_ctx, "snap", nullptr)); + + uint64_t size = object_size - 1; + librbd::NoOpProgressContext no_op_progress_ctx; + ASSERT_EQ(0, m_remote_image_ctx->operations->resize(size, true, + no_op_progress_ctx)); + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + bufferlist read_remote_bl; + read_remote_bl.append(std::string(len, '\0')); + bufferlist read_local_bl; + read_local_bl.append(std::string(len, '\0')); + + ASSERT_LE(0, m_remote_image_ctx->io_work_queue->read( + off, len, librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, m_local_image_ctx->io_work_queue->read( + off, len, librbd::io::ReadResult{&read_local_bl}, 0)); + + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); +} + +TEST_F(TestImageSync, Discard) { + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + + uint64_t off = 0; + uint64_t len = object_size / 10; + + bufferlist bl; + bl.append(std::string(len, '1')); + ASSERT_EQ((int)len, m_remote_image_ctx->io_work_queue->write(off, len, + std::move(bl), + 0)); + { + RWLock::RLocker owner_locker(m_remote_image_ctx->owner_lock); + ASSERT_EQ(0, flush(m_remote_image_ctx)); + } + + ASSERT_EQ(0, create_snap(m_remote_image_ctx, "snap", nullptr)); + + ASSERT_EQ((int)len - 2, + m_remote_image_ctx->io_work_queue->discard( + off + 1, len - 2, m_remote_image_ctx->discard_granularity_bytes)); + { + RWLock::RLocker owner_locker(m_remote_image_ctx->owner_lock); + ASSERT_EQ(0, flush(m_remote_image_ctx)); + } + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + bufferlist read_remote_bl; + read_remote_bl.append(std::string(object_size, '\0')); + bufferlist read_local_bl; + read_local_bl.append(std::string(object_size, '\0')); + + ASSERT_LE(0, m_remote_image_ctx->io_work_queue->read( + off, len, librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, m_local_image_ctx->io_work_queue->read( + off, len, librbd::io::ReadResult{&read_local_bl}, 0)); + + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); +} + +TEST_F(TestImageSync, SnapshotStress) { + std::list<std::string> snap_names; + + const int num_snaps = 4; + for (int idx = 0; idx <= num_snaps; ++idx) { + scribble(m_remote_image_ctx, 10, 102400); + + librbd::NoOpProgressContext no_op_progress_ctx; + uint64_t size = 1 + rand() % m_image_size; + ASSERT_EQ(0, m_remote_image_ctx->operations->resize(size, true, + no_op_progress_ctx)); + ASSERT_EQ(0, m_remote_image_ctx->state->refresh()); + + if (idx < num_snaps) { + snap_names.push_back("snap" + stringify(idx + 1)); + ASSERT_EQ(0, create_snap(m_remote_image_ctx, snap_names.back().c_str(), + nullptr)); + } else { + snap_names.push_back(""); + } + } + + C_SaferCond ctx; + ImageSync<> *request = create_request(&ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); + + int64_t object_size = std::min<int64_t>( + m_remote_image_ctx->size, 1 << m_remote_image_ctx->order); + bufferlist read_remote_bl; + read_remote_bl.append(std::string(object_size, '1')); + bufferlist read_local_bl; + read_local_bl.append(std::string(object_size, '1')); + + for (auto &snap_name : snap_names) { + uint64_t remote_snap_id; + { + RWLock::RLocker remote_snap_locker(m_remote_image_ctx->snap_lock); + remote_snap_id = m_remote_image_ctx->get_snap_id( + cls::rbd::UserSnapshotNamespace{}, snap_name); + } + + uint64_t remote_size; + { + C_SaferCond ctx; + m_remote_image_ctx->state->snap_set(remote_snap_id, &ctx); + ASSERT_EQ(0, ctx.wait()); + + RWLock::RLocker remote_snap_locker(m_remote_image_ctx->snap_lock); + remote_size = m_remote_image_ctx->get_image_size( + m_remote_image_ctx->snap_id); + } + + uint64_t local_snap_id; + { + RWLock::RLocker snap_locker(m_local_image_ctx->snap_lock); + local_snap_id = m_local_image_ctx->get_snap_id( + cls::rbd::UserSnapshotNamespace{}, snap_name); + } + + uint64_t local_size; + { + C_SaferCond ctx; + m_local_image_ctx->state->snap_set(local_snap_id, &ctx); + ASSERT_EQ(0, ctx.wait()); + + RWLock::RLocker snap_locker(m_local_image_ctx->snap_lock); + local_size = m_local_image_ctx->get_image_size( + m_local_image_ctx->snap_id); + bool flags_set; + ASSERT_EQ(0, m_local_image_ctx->test_flags(m_local_image_ctx->snap_id, + RBD_FLAG_OBJECT_MAP_INVALID, + m_local_image_ctx->snap_lock, + &flags_set)); + ASSERT_FALSE(flags_set); + } + + ASSERT_EQ(remote_size, local_size); + + for (uint64_t offset = 0; offset < remote_size; offset += object_size) { + ASSERT_LE(0, m_remote_image_ctx->io_work_queue->read( + offset, object_size, + librbd::io::ReadResult{&read_remote_bl}, 0)); + ASSERT_LE(0, m_local_image_ctx->io_work_queue->read( + offset, object_size, + librbd::io::ReadResult{&read_local_bl}, 0)); + ASSERT_TRUE(read_remote_bl.contents_equal(read_local_bl)); + } + } +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_InstanceWatcher.cc b/src/test/rbd_mirror/test_InstanceWatcher.cc new file mode 100644 index 00000000..92a8f943 --- /dev/null +++ b/src/test/rbd_mirror/test_InstanceWatcher.cc @@ -0,0 +1,132 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "include/stringify.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/Utils.h" +#include "librbd/internal.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "common/Cond.h" + +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" + +using rbd::mirror::InstanceWatcher; + +void register_test_instance_watcher() { +} + +class TestInstanceWatcher : public ::rbd::mirror::TestFixture { +public: + std::string m_instance_id; + std::string m_oid; + + void SetUp() override { + TestFixture::SetUp(); + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); + + m_instance_id = stringify(m_local_io_ctx.get_instance_id()); + m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id; + } + + void get_instances(std::vector<std::string> *instance_ids) { + instance_ids->clear(); + C_SaferCond on_get; + InstanceWatcher<>::get_instances(m_local_io_ctx, instance_ids, &on_get); + EXPECT_EQ(0, on_get.wait()); + } +}; + +TEST_F(TestInstanceWatcher, InitShutdown) +{ + InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue, + nullptr, m_instance_id); + std::vector<std::string> instance_ids; + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + uint64_t size; + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr)); + + // Init + ASSERT_EQ(0, instance_watcher.init()); + + get_instances(&instance_ids); + ASSERT_EQ(1U, instance_ids.size()); + ASSERT_EQ(m_instance_id, instance_ids[0]); + + ASSERT_EQ(0, m_local_io_ctx.stat(m_oid, &size, nullptr)); + std::list<obj_watch_t> watchers; + ASSERT_EQ(0, m_local_io_ctx.list_watchers(m_oid, &watchers)); + ASSERT_EQ(1U, watchers.size()); + ASSERT_EQ(m_instance_id, stringify(watchers.begin()->watcher_id)); + + get_instances(&instance_ids); + ASSERT_EQ(1U, instance_ids.size()); + + // Shutdown + instance_watcher.shut_down(); + + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr)); + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); +} + +TEST_F(TestInstanceWatcher, Remove) +{ + std::string instance_id = "instance_id"; + std::string oid = RBD_MIRROR_INSTANCE_PREFIX + instance_id; + + std::vector<std::string> instance_ids; + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + uint64_t size; + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(oid, &size, nullptr)); + + librados::Rados cluster; + librados::IoCtx io_ctx; + ASSERT_EQ("", connect_cluster_pp(cluster)); + ASSERT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx)); + InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue, + nullptr, "instance_id"); + // Init + ASSERT_EQ(0, instance_watcher.init()); + + get_instances(&instance_ids); + ASSERT_EQ(1U, instance_ids.size()); + ASSERT_EQ(instance_id, instance_ids[0]); + + ASSERT_EQ(0, m_local_io_ctx.stat(oid, &size, nullptr)); + std::list<obj_watch_t> watchers; + ASSERT_EQ(0, m_local_io_ctx.list_watchers(oid, &watchers)); + ASSERT_EQ(1U, watchers.size()); + + // Remove + C_SaferCond on_remove; + InstanceWatcher<>::remove_instance(m_local_io_ctx, m_threads->work_queue, + "instance_id", &on_remove); + ASSERT_EQ(0, on_remove.wait()); + + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(oid, &size, nullptr)); + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + // Shutdown + instance_watcher.shut_down(); + + ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr)); + get_instances(&instance_ids); + ASSERT_EQ(0U, instance_ids.size()); + + // Remove NOENT + C_SaferCond on_remove_noent; + InstanceWatcher<>::remove_instance(m_local_io_ctx, m_threads->work_queue, + instance_id, &on_remove_noent); + ASSERT_EQ(0, on_remove_noent.wait()); +} diff --git a/src/test/rbd_mirror/test_Instances.cc b/src/test/rbd_mirror/test_Instances.cc new file mode 100644 index 00000000..c4e8bd30 --- /dev/null +++ b/src/test/rbd_mirror/test_Instances.cc @@ -0,0 +1,165 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_client.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Instances.h" +#include "tools/rbd_mirror/Threads.h" +#include "common/Cond.h" + +#include "test/librados/test.h" +#include "gtest/gtest.h" +#include <vector> + +using rbd::mirror::InstanceWatcher; +using rbd::mirror::Instances; + +void register_test_instances() { +} + +class TestInstances : public ::rbd::mirror::TestFixture { +public: + struct Listener : public rbd::mirror::instances::Listener { + std::mutex lock; + + struct Instance { + uint32_t count = 0; + std::set<std::string> ids; + C_SaferCond ctx; + }; + + Instance add; + Instance remove; + + void handle(const InstanceIds& instance_ids, Instance* instance) { + std::unique_lock<std::mutex> locker(lock); + for (auto& instance_id : instance_ids) { + ceph_assert(instance->count > 0); + --instance->count; + + instance->ids.insert(instance_id); + if (instance->count == 0) { + instance->ctx.complete(0); + } + } + } + + void handle_added(const InstanceIds& instance_ids) override { + handle(instance_ids, &add); + } + + void handle_removed(const InstanceIds& instance_ids) override { + handle(instance_ids, &remove); + } + }; + + virtual void SetUp() { + TestFixture::SetUp(); + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); + + m_instance_id = stringify(m_local_io_ctx.get_instance_id()); + } + + Listener m_listener; + std::string m_instance_id; +}; + +TEST_F(TestInstances, InitShutdown) +{ + m_listener.add.count = 1; + Instances<> instances(m_threads, m_local_io_ctx, m_instance_id, m_listener); + + std::string instance_id = "instance_id"; + ASSERT_EQ(0, librbd::cls_client::mirror_instances_add(&m_local_io_ctx, + instance_id)); + + C_SaferCond on_init; + instances.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + ASSERT_LT(0U, m_listener.add.count); + instances.unblock_listener(); + + ASSERT_EQ(0, m_listener.add.ctx.wait()); + ASSERT_EQ(std::set<std::string>({instance_id}), m_listener.add.ids); + + C_SaferCond on_shut_down; + instances.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} + +TEST_F(TestInstances, InitEnoent) +{ + Instances<> instances(m_threads, m_local_io_ctx, m_instance_id, m_listener); + + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + + C_SaferCond on_init; + instances.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + C_SaferCond on_shut_down; + instances.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} + +TEST_F(TestInstances, NotifyRemove) +{ + // speed testing up a little + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", "1")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_missed_heartbeats", + "2")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_acquire_attempts_before_break", + "0")); + + m_listener.add.count = 2; + m_listener.remove.count = 1; + Instances<> instances(m_threads, m_local_io_ctx, m_instance_id, m_listener); + + std::string instance_id1 = "instance_id1"; + std::string instance_id2 = "instance_id2"; + + ASSERT_EQ(0, librbd::cls_client::mirror_instances_add(&m_local_io_ctx, + instance_id1)); + + + C_SaferCond on_init; + instances.init(&on_init); + ASSERT_EQ(0, on_init.wait()); + + instances.acked({instance_id1, instance_id2}); + + ASSERT_LT(0U, m_listener.add.count); + instances.unblock_listener(); + + ASSERT_EQ(0, m_listener.add.ctx.wait()); + ASSERT_EQ(std::set<std::string>({instance_id1, instance_id2}), + m_listener.add.ids); + + std::vector<std::string> instance_ids; + for (int i = 0; i < 100; i++) { + instances.acked({instance_id1}); + if (m_listener.remove.count > 0) { + usleep(250000); + } + } + + instances.acked({instance_id1}); + ASSERT_EQ(0, m_listener.remove.ctx.wait()); + ASSERT_EQ(std::set<std::string>({instance_id2}), + m_listener.remove.ids); + + C_SaferCond on_get; + instances.acked({instance_id1}); + InstanceWatcher<>::get_instances(m_local_io_ctx, &instance_ids, &on_get); + EXPECT_EQ(0, on_get.wait()); + EXPECT_EQ(1U, instance_ids.size()); + ASSERT_EQ(instance_ids[0], instance_id1); + + C_SaferCond on_shut_down; + instances.shut_down(&on_shut_down); + ASSERT_EQ(0, on_shut_down.wait()); +} diff --git a/src/test/rbd_mirror/test_LeaderWatcher.cc b/src/test/rbd_mirror/test_LeaderWatcher.cc new file mode 100644 index 00000000..8a5cd890 --- /dev/null +++ b/src/test/rbd_mirror/test_LeaderWatcher.cc @@ -0,0 +1,317 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "librbd/internal.h" +#include "librbd/Utils.h" +#include "librbd/api/Mirror.h" +#include "test/librbd/test_support.h" +#include "test/rbd_mirror/test_fixture.h" +#include "tools/rbd_mirror/LeaderWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "common/Cond.h" + +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" + +using librbd::util::unique_lock_name; +using rbd::mirror::LeaderWatcher; + +void register_test_leader_watcher() { +} + +class TestLeaderWatcher : public ::rbd::mirror::TestFixture { +public: + class Listener : public rbd::mirror::leader_watcher::Listener { + public: + Listener() + : m_test_lock(unique_lock_name("LeaderWatcher::m_test_lock", this)) { + } + + void on_acquire(int r, Context *ctx) { + Mutex::Locker locker(m_test_lock); + m_on_acquire_r = r; + m_on_acquire = ctx; + } + + void on_release(int r, Context *ctx) { + Mutex::Locker locker(m_test_lock); + m_on_release_r = r; + m_on_release = ctx; + } + + int acquire_count() const { + Mutex::Locker locker(m_test_lock); + return m_acquire_count; + } + + int release_count() const { + Mutex::Locker locker(m_test_lock); + return m_release_count; + } + + void post_acquire_handler(Context *on_finish) override { + Mutex::Locker locker(m_test_lock); + m_acquire_count++; + on_finish->complete(m_on_acquire_r); + m_on_acquire_r = 0; + if (m_on_acquire != nullptr) { + m_on_acquire->complete(0); + m_on_acquire = nullptr; + } + } + + void pre_release_handler(Context *on_finish) override { + Mutex::Locker locker(m_test_lock); + m_release_count++; + on_finish->complete(m_on_release_r); + m_on_release_r = 0; + if (m_on_release != nullptr) { + m_on_release->complete(0); + m_on_release = nullptr; + } + } + + void update_leader_handler(const std::string &leader_instance_id) override { + } + + void handle_instances_added(const InstanceIds& instance_ids) override { + } + void handle_instances_removed(const InstanceIds& instance_ids) override { + } + + private: + mutable Mutex m_test_lock; + int m_acquire_count = 0; + int m_release_count = 0; + int m_on_acquire_r = 0; + int m_on_release_r = 0; + Context *m_on_acquire = nullptr; + Context *m_on_release = nullptr; + }; + + struct Connection { + librados::Rados cluster; + librados::IoCtx io_ctx; + }; + + std::list<std::unique_ptr<Connection> > m_connections; + + void SetUp() override { + TestFixture::SetUp(); + EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_local_io_ctx, + RBD_MIRROR_MODE_POOL)); + + if (is_librados_test_stub(*_rados)) { + // speed testing up a little + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", + "1")); + } + } + + librados::IoCtx &create_connection(bool no_heartbeats = false) { + m_connections.push_back(std::unique_ptr<Connection>(new Connection())); + Connection *c = m_connections.back().get(); + + EXPECT_EQ("", connect_cluster_pp(c->cluster)); + if (no_heartbeats) { + EXPECT_EQ(0, c->cluster.conf_set("rbd_mirror_leader_heartbeat_interval", + "3600")); + } else if (is_librados_test_stub(*_rados)) { + EXPECT_EQ(0, c->cluster.conf_set("rbd_mirror_leader_heartbeat_interval", + "1")); + } + EXPECT_EQ(0, c->cluster.ioctx_create(_local_pool_name.c_str(), c->io_ctx)); + + return c->io_ctx; + } +}; + +TEST_F(TestLeaderWatcher, InitShutdown) +{ + Listener listener; + LeaderWatcher<> leader_watcher(m_threads, m_local_io_ctx, &listener); + + C_SaferCond on_init_acquire; + listener.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + leader_watcher.shut_down(); + ASSERT_EQ(1, listener.acquire_count()); + ASSERT_EQ(1, listener.release_count()); + ASSERT_FALSE(leader_watcher.is_leader()); +} + +TEST_F(TestLeaderWatcher, Release) +{ + Listener listener; + LeaderWatcher<> leader_watcher(m_threads, m_local_io_ctx, &listener); + + C_SaferCond on_init_acquire; + listener.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + C_SaferCond on_release; + C_SaferCond on_acquire; + listener.on_release(0, &on_release); + listener.on_acquire(0, &on_acquire); + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release.wait()); + ASSERT_FALSE(leader_watcher.is_leader()); + + // wait for lock re-acquired due to no another locker + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + C_SaferCond on_release2; + listener.on_release(0, &on_release2); + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release2.wait()); + + leader_watcher.shut_down(); + ASSERT_EQ(2, listener.acquire_count()); + ASSERT_EQ(2, listener.release_count()); +} + +TEST_F(TestLeaderWatcher, ListenerError) +{ + Listener listener; + LeaderWatcher<> leader_watcher(m_threads, m_local_io_ctx, &listener); + + // make listener return error on acquire + C_SaferCond on_init_acquire, on_init_release; + listener.on_acquire(-EINVAL, &on_init_acquire); + listener.on_release(0, &on_init_release); + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + ASSERT_EQ(0, on_init_release.wait()); + ASSERT_FALSE(leader_watcher.is_leader()); + + // wait for lock re-acquired due to no another locker + C_SaferCond on_acquire; + listener.on_acquire(0, &on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher.is_leader()); + + // make listener return error on release + C_SaferCond on_release; + listener.on_release(-EINVAL, &on_release); + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release.wait()); + ASSERT_FALSE(leader_watcher.is_leader()); + + leader_watcher.shut_down(); + ASSERT_EQ(2, listener.acquire_count()); + ASSERT_EQ(2, listener.release_count()); + ASSERT_FALSE(leader_watcher.is_leader()); +} + +TEST_F(TestLeaderWatcher, Two) +{ + Listener listener1; + LeaderWatcher<> leader_watcher1(m_threads, create_connection(), &listener1); + + C_SaferCond on_init_acquire; + listener1.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher1.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + + Listener listener2; + LeaderWatcher<> leader_watcher2(m_threads, create_connection(), &listener2); + + ASSERT_EQ(0, leader_watcher2.init()); + ASSERT_TRUE(leader_watcher1.is_leader()); + ASSERT_FALSE(leader_watcher2.is_leader()); + + C_SaferCond on_release; + C_SaferCond on_acquire; + listener1.on_release(0, &on_release); + listener2.on_acquire(0, &on_acquire); + leader_watcher1.release_leader(); + ASSERT_EQ(0, on_release.wait()); + ASSERT_FALSE(leader_watcher1.is_leader()); + + // wait for lock acquired by another watcher + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher2.is_leader()); + + leader_watcher1.shut_down(); + leader_watcher2.shut_down(); + + ASSERT_EQ(1, listener1.acquire_count()); + ASSERT_EQ(1, listener1.release_count()); + ASSERT_EQ(1, listener2.acquire_count()); + ASSERT_EQ(1, listener2.release_count()); +} + +TEST_F(TestLeaderWatcher, Break) +{ + Listener listener1, listener2; + LeaderWatcher<> leader_watcher1(m_threads, + create_connection(true /* no heartbeats */), + &listener1); + LeaderWatcher<> leader_watcher2(m_threads, create_connection(), &listener2); + + C_SaferCond on_init_acquire; + listener1.on_acquire(0, &on_init_acquire); + ASSERT_EQ(0, leader_watcher1.init()); + ASSERT_EQ(0, on_init_acquire.wait()); + + C_SaferCond on_acquire; + listener2.on_acquire(0, &on_acquire); + ASSERT_EQ(0, leader_watcher2.init()); + ASSERT_FALSE(leader_watcher2.is_leader()); + + // wait for lock broken due to no heartbeats and re-acquired + ASSERT_EQ(0, on_acquire.wait()); + ASSERT_TRUE(leader_watcher2.is_leader()); + + leader_watcher1.shut_down(); + leader_watcher2.shut_down(); +} + +TEST_F(TestLeaderWatcher, Stress) +{ + const int WATCHERS_COUNT = 20; + std::list<LeaderWatcher<> *> leader_watchers; + Listener listener; + + for (int i = 0; i < WATCHERS_COUNT; i++) { + auto leader_watcher = + new LeaderWatcher<>(m_threads, create_connection(), &listener); + leader_watchers.push_back(leader_watcher); + } + + C_SaferCond on_init_acquire; + listener.on_acquire(0, &on_init_acquire); + for (auto &leader_watcher : leader_watchers) { + ASSERT_EQ(0, leader_watcher->init()); + } + ASSERT_EQ(0, on_init_acquire.wait()); + + while (true) { + C_SaferCond on_acquire; + listener.on_acquire(0, &on_acquire); + std::unique_ptr<LeaderWatcher<> > leader_watcher; + for (auto it = leader_watchers.begin(); it != leader_watchers.end(); ) { + if ((*it)->is_leader()) { + ASSERT_FALSE(leader_watcher); + leader_watcher.reset(*it); + it = leader_watchers.erase(it); + } else { + it++; + } + } + + ASSERT_TRUE(leader_watcher); + leader_watcher->shut_down(); + if (leader_watchers.empty()) { + break; + } + ASSERT_EQ(0, on_acquire.wait()); + } +} diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc new file mode 100644 index 00000000..108dc355 --- /dev/null +++ b/src/test/rbd_mirror/test_PoolWatcher.cc @@ -0,0 +1,254 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/rados/librados.hpp" +#include "include/rbd/librbd.hpp" +#include "include/stringify.h" +#include "test/rbd_mirror/test_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "include/rbd_types.h" +#include "librbd/internal.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/api/Mirror.h" +#include "common/Cond.h" +#include "common/errno.h" +#include "common/Mutex.h" +#include "tools/rbd_mirror/PoolWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" +#include "tools/rbd_mirror/pool_watcher/Types.h" +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" +#include <boost/scope_exit.hpp> +#include <iostream> +#include <map> +#include <memory> +#include <set> +#include <vector> + +using rbd::mirror::ImageId; +using rbd::mirror::ImageIds; +using rbd::mirror::PoolWatcher; +using rbd::mirror::PeerSpec; +using rbd::mirror::RadosRef; +using std::map; +using std::set; +using std::string; + +void register_test_pool_watcher() { +} + +class TestPoolWatcher : public ::rbd::mirror::TestFixture { +public: + + TestPoolWatcher() + : m_lock("TestPoolWatcherLock"), m_pool_watcher_listener(this), + m_image_number(0), m_snap_number(0) + { + m_cluster = std::make_shared<librados::Rados>(); + EXPECT_EQ("", connect_cluster_pp(*m_cluster)); + } + + void TearDown() override { + if (m_pool_watcher) { + C_SaferCond ctx; + m_pool_watcher->shut_down(&ctx); + EXPECT_EQ(0, ctx.wait()); + } + + m_cluster->wait_for_latest_osdmap(); + for (auto& pool : m_pools) { + EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str())); + } + + TestFixture::TearDown(); + } + + struct PoolWatcherListener : public rbd::mirror::pool_watcher::Listener { + TestPoolWatcher *test; + Cond cond; + ImageIds image_ids; + + explicit PoolWatcherListener(TestPoolWatcher *test) : test(test) { + } + + void handle_update(const std::string &mirror_uuid, + ImageIds &&added_image_ids, + ImageIds &&removed_image_ids) override { + Mutex::Locker locker(test->m_lock); + for (auto &image_id : removed_image_ids) { + image_ids.erase(image_id); + } + image_ids.insert(added_image_ids.begin(), added_image_ids.end()); + cond.Signal(); + } + }; + + void create_pool(bool enable_mirroring, const PeerSpec &peer, string *name=nullptr) { + string pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str())); + + int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str()); + ASSERT_GE(pool_id, 0); + m_pools.insert(pool_name); + + librados::IoCtx ioctx; + ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx)); + ioctx.application_enable("rbd", true); + + m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx, + m_pool_watcher_listener)); + + if (enable_mirroring) { + ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx, + RBD_MIRROR_MODE_POOL)); + std::string uuid; + ASSERT_EQ(0, librbd::api::Mirror<>::peer_add(ioctx, &uuid, + peer.cluster_name, + peer.client_name)); + } + if (name != nullptr) { + *name = pool_name; + } + + m_pool_watcher->init(); + } + + string get_image_id(librados::IoCtx *ioctx, const string &image_name) { + string obj = librbd::util::id_obj_name(image_name); + string id; + EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id)); + return id; + } + + void create_image(const string &pool_name, bool mirrored=true, + string *image_name=nullptr) { + uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); + string name = "image" + stringify(++m_image_number); + if (mirrored) { + features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; + } + + librados::IoCtx ioctx; + ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx)); + int order = 0; + ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false, + features, &order, 0, 0)); + if (mirrored) { + librbd::Image image; + librbd::RBD rbd; + rbd.open(ioctx, image, name.c_str()); + image.mirror_image_enable(); + + librbd::mirror_image_info_t mirror_image_info; + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, + sizeof(mirror_image_info))); + image.close(); + + m_mirrored_images.insert(ImageId( + mirror_image_info.global_id, get_image_id(&ioctx, name))); + } + if (image_name != nullptr) + *image_name = name; + } + + void clone_image(const string &parent_pool_name, + const string &parent_image_name, + const string &clone_pool_name, + bool mirrored=true, + string *image_name=nullptr) { + librados::IoCtx pioctx, cioctx; + ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx)); + ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx)); + + string snap_name = "snap" + stringify(++m_snap_number); + { + librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(), + "", "", pioctx, false); + ictx->state->open(0); + EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + snap_name.c_str())); + EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(), + snap_name.c_str())); + ictx->state->close(); + } + + uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context); + string name = "clone" + stringify(++m_image_number); + if (mirrored) { + features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING; + } + int order = 0; + librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(), + cioctx, name.c_str(), features, &order, 0, 0); + if (mirrored) { + librbd::Image image; + librbd::RBD rbd; + rbd.open(cioctx, image, name.c_str()); + image.mirror_image_enable(); + + librbd::mirror_image_info_t mirror_image_info; + ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info, + sizeof(mirror_image_info))); + image.close(); + + m_mirrored_images.insert(ImageId( + mirror_image_info.global_id, get_image_id(&cioctx, name))); + } + if (image_name != nullptr) + *image_name = name; + } + + void check_images() { + Mutex::Locker l(m_lock); + while (m_mirrored_images != m_pool_watcher_listener.image_ids) { + if (m_pool_watcher_listener.cond.WaitInterval( + m_lock, utime_t(10, 0)) != 0) { + break; + } + } + + ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids); + } + + Mutex m_lock; + RadosRef m_cluster; + PoolWatcherListener m_pool_watcher_listener; + unique_ptr<PoolWatcher<> > m_pool_watcher; + + set<string> m_pools; + ImageIds m_mirrored_images; + + uint64_t m_image_number; + uint64_t m_snap_number; +}; + +TEST_F(TestPoolWatcher, EmptyPool) { + string uuid1 = "00000000-0000-0000-0000-000000000001"; + PeerSpec site1(uuid1, "site1", "mirror1"); + create_pool(true, site1); + check_images(); +} + +TEST_F(TestPoolWatcher, ReplicatedPools) { + string uuid1 = "00000000-0000-0000-0000-000000000001"; + PeerSpec site1(uuid1, "site1", "mirror1"); + string first_pool, local_pool, last_pool; + create_pool(true, site1, &first_pool); + check_images(); + create_image(first_pool); + check_images(); + string parent_image, parent_image2; + create_image(first_pool, true, &parent_image); + check_images(); + clone_image(first_pool, parent_image, first_pool); + check_images(); + clone_image(first_pool, parent_image, first_pool, true, &parent_image2); + check_images(); + create_image(first_pool, false); + check_images(); +} diff --git a/src/test/rbd_mirror/test_fixture.cc b/src/test/rbd_mirror/test_fixture.cc new file mode 100644 index 00000000..b0b345fb --- /dev/null +++ b/src/test/rbd_mirror/test_fixture.cc @@ -0,0 +1,160 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "cls/rbd/cls_rbd_types.h" +#include "test/rbd_mirror/test_fixture.h" +#include "include/stringify.h" +#include "include/rbd/librbd.hpp" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "test/librados/test_cxx.h" +#include "tools/rbd_mirror/Threads.h" + +namespace rbd { +namespace mirror { + +std::string TestFixture::_local_pool_name; +std::string TestFixture::_remote_pool_name; +std::shared_ptr<librados::Rados> TestFixture::_rados; +uint64_t TestFixture::_image_number = 0; +std::string TestFixture::_data_pool; + +TestFixture::TestFixture() { +} + +void TestFixture::SetUpTestCase() { + _rados = std::shared_ptr<librados::Rados>(new librados::Rados()); + ASSERT_EQ("", connect_cluster_pp(*_rados.get())); + ASSERT_EQ(0, _rados->conf_set("rbd_cache", "false")); + + _local_pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, _rados->pool_create(_local_pool_name.c_str())); + + librados::IoCtx local_ioctx; + ASSERT_EQ(0, _rados->ioctx_create(_local_pool_name.c_str(), local_ioctx)); + local_ioctx.application_enable("rbd", true); + + _remote_pool_name = get_temp_pool_name("test-rbd-mirror-"); + ASSERT_EQ(0, _rados->pool_create(_remote_pool_name.c_str())); + + librados::IoCtx remote_ioctx; + ASSERT_EQ(0, _rados->ioctx_create(_remote_pool_name.c_str(), remote_ioctx)); + remote_ioctx.application_enable("rbd", true); + + ASSERT_EQ(0, create_image_data_pool(_data_pool)); + if (!_data_pool.empty()) { + printf("using image data pool: %s\n", _data_pool.c_str()); + } +} + +void TestFixture::TearDownTestCase() { + if (!_data_pool.empty()) { + ASSERT_EQ(0, _rados->pool_delete(_data_pool.c_str())); + } + + ASSERT_EQ(0, _rados->pool_delete(_remote_pool_name.c_str())); + ASSERT_EQ(0, _rados->pool_delete(_local_pool_name.c_str())); + _rados->shutdown(); +} + +void TestFixture::SetUp() { + static bool seeded = false; + if (!seeded) { + seeded = true; + int seed = getpid(); + cout << "seed " << seed << std::endl; + srand(seed); + } + + ASSERT_EQ(0, _rados->ioctx_create(_local_pool_name.c_str(), m_local_io_ctx)); + ASSERT_EQ(0, _rados->ioctx_create(_remote_pool_name.c_str(), m_remote_io_ctx)); + m_image_name = get_temp_image_name(); + + m_threads = new rbd::mirror::Threads<>(reinterpret_cast<CephContext*>( + m_local_io_ctx.cct())); +} + +void TestFixture::TearDown() { + for (auto image_ctx : m_image_ctxs) { + image_ctx->state->close(); + } + + m_remote_io_ctx.close(); + m_local_io_ctx.close(); + + delete m_threads; +} + +int TestFixture::create_image(librbd::RBD &rbd, librados::IoCtx &ioctx, + const std::string &name, uint64_t size) { + int order = 18; + return rbd.create2(ioctx, name.c_str(), size, RBD_FEATURES_ALL, &order); +} + +int TestFixture::open_image(librados::IoCtx &io_ctx, + const std::string &image_name, + librbd::ImageCtx **image_ctx) { + *image_ctx = new librbd::ImageCtx(image_name.c_str(), "", nullptr, io_ctx, + false); + m_image_ctxs.insert(*image_ctx); + return (*image_ctx)->state->open(0); +} + +int TestFixture::create_snap(librbd::ImageCtx *image_ctx, const char* snap_name, + librados::snap_t *snap_id) { + int r = image_ctx->operations->snap_create(cls::rbd::UserSnapshotNamespace(), + snap_name); + if (r < 0) { + return r; + } + + r = image_ctx->state->refresh(); + if (r < 0) { + return r; + } + + if (image_ctx->snap_ids.count({cls::rbd::UserSnapshotNamespace(), + snap_name}) == 0) { + return -ENOENT; + } + + if (snap_id != nullptr) { + *snap_id = image_ctx->snap_ids[{cls::rbd::UserSnapshotNamespace(), + snap_name}]; + } + return 0; +} + +std::string TestFixture::get_temp_image_name() { + ++_image_number; + return "image" + stringify(_image_number); +} + +int TestFixture::create_image_data_pool(std::string &data_pool) { + std::string pool; + int r = _rados->conf_get("rbd_default_data_pool", pool); + if (r != 0) { + return r; + } else if (pool.empty()) { + return 0; + } + + r = _rados->pool_create(pool.c_str()); + if (r < 0) { + return r; + } + + librados::IoCtx data_ioctx; + r = _rados->ioctx_create(pool.c_str(), data_ioctx); + if (r < 0) { + return r; + } + + data_ioctx.application_enable("rbd", true); + data_pool = pool; + return 0; +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_fixture.h b/src/test/rbd_mirror/test_fixture.h new file mode 100644 index 00000000..217ae810 --- /dev/null +++ b/src/test/rbd_mirror/test_fixture.h @@ -0,0 +1,65 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_TEST_RBD_MIRROR_TEST_FIXTURE_H +#define CEPH_TEST_RBD_MIRROR_TEST_FIXTURE_H + +#include "include/int_types.h" +#include "include/rados/librados.hpp" +#include <gtest/gtest.h> +#include <memory> +#include <set> + +namespace librbd { +class ImageCtx; +class RBD; +} + +namespace rbd { +namespace mirror { + +template <typename> class Threads; + +class TestFixture : public ::testing::Test { +public: + TestFixture(); + + static void SetUpTestCase(); + static void TearDownTestCase(); + + void SetUp() override; + void TearDown() override; + + librados::IoCtx m_local_io_ctx; + librados::IoCtx m_remote_io_ctx; + + std::string m_image_name; + uint64_t m_image_size = 1 << 24; + + std::set<librbd::ImageCtx *> m_image_ctxs; + + Threads<librbd::ImageCtx> *m_threads = nullptr; + + + int create_image(librbd::RBD &rbd, librados::IoCtx &ioctx, + const std::string &name, uint64_t size); + int open_image(librados::IoCtx &io_ctx, const std::string &image_name, + librbd::ImageCtx **image_ctx); + + int create_snap(librbd::ImageCtx *image_ctx, const char* snap_name, + librados::snap_t *snap_id = nullptr); + + static std::string get_temp_image_name(); + static int create_image_data_pool(std::string &data_pool); + + static std::string _local_pool_name; + static std::string _remote_pool_name; + static std::shared_ptr<librados::Rados> _rados; + static uint64_t _image_number; + static std::string _data_pool; +}; + +} // namespace mirror +} // namespace rbd + +#endif // CEPH_TEST_RBD_MIRROR_TEST_FIXTURE_H diff --git a/src/test/rbd_mirror/test_main.cc b/src/test/rbd_mirror/test_main.cc new file mode 100644 index 00000000..9226b086 --- /dev/null +++ b/src/test/rbd_mirror/test_main.cc @@ -0,0 +1,53 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/perf_counters.h" +#include "include/rados/librados.hpp" +#include "global/global_context.h" +#include "test/librados/test_cxx.h" +#include "gtest/gtest.h" +#include <iostream> +#include <string> + +PerfCounters *g_perf_counters = nullptr; + +extern void register_test_cluster_watcher(); +extern void register_test_image_policy(); +extern void register_test_image_sync(); +extern void register_test_instance_watcher(); +extern void register_test_instances(); +extern void register_test_leader_watcher(); +extern void register_test_pool_watcher(); +extern void register_test_rbd_mirror(); +extern void register_test_rbd_mirror_image_deleter(); + +int main(int argc, char **argv) +{ + register_test_cluster_watcher(); + register_test_image_policy(); + register_test_image_sync(); + register_test_instance_watcher(); + register_test_instances(); + register_test_leader_watcher(); + register_test_pool_watcher(); + register_test_rbd_mirror(); + register_test_rbd_mirror_image_deleter(); + + ::testing::InitGoogleTest(&argc, argv); + + librados::Rados rados; + std::string result = connect_cluster_pp(rados); + if (result != "" ) { + std::cerr << result << std::endl; + return 1; + } + + g_ceph_context = reinterpret_cast<CephContext*>(rados.cct()); + + int r = rados.conf_set("lockdep", "true"); + if (r < 0) { + std::cerr << "failed to enable lockdep" << std::endl; + return -r; + } + return RUN_ALL_TESTS(); +} diff --git a/src/test/rbd_mirror/test_mock_ImageMap.cc b/src/test/rbd_mirror/test_mock_ImageMap.cc new file mode 100644 index 00000000..e223b8b9 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_ImageMap.cc @@ -0,0 +1,1591 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "librbd/MirroringWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/ImageMap.h" +#include "tools/rbd_mirror/image_map/LoadRequest.h" +#include "tools/rbd_mirror/image_map/UpdateRequest.h" +#include "tools/rbd_mirror/image_map/Types.h" +#include "include/stringify.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + Mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +namespace image_map { + +template <> +struct LoadRequest<librbd::MockTestImageCtx> { + std::map<std::string, cls::rbd::MirrorImageMap> *image_map; + Context *on_finish = nullptr; + + static LoadRequest *s_instance; + static LoadRequest *create(librados::IoCtx &ioctx, + std::map<std::string, cls::rbd::MirrorImageMap> *image_map, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_map = image_map; + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + LoadRequest() { + s_instance = this; + } +}; + +template <> +struct UpdateRequest<librbd::MockTestImageCtx> { + Context *on_finish = nullptr; + static UpdateRequest *s_instance; + static UpdateRequest *create(librados::IoCtx &ioctx, + std::map<std::string, cls::rbd::MirrorImageMap> &&update_mapping, + std::set<std::string> &&global_image_ids, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + UpdateRequest() { + s_instance = this; + } +}; + +LoadRequest<librbd::MockTestImageCtx> * +LoadRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +UpdateRequest<librbd::MockTestImageCtx> * +UpdateRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_map + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/ImageMap.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::WithArg; +using ::testing::AtLeast; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::ReturnArg; +using ::testing::StrEq; + +using image_map::Listener; +using image_map::LoadRequest; +using image_map::UpdateRequest; + +using ::rbd::mirror::Threads; + +class TestMockImageMap : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef ImageMap<librbd::MockTestImageCtx> MockImageMap; + typedef LoadRequest<librbd::MockTestImageCtx> MockLoadRequest; + typedef UpdateRequest<librbd::MockTestImageCtx> MockUpdateRequest; + + struct MockListener : Listener { + TestMockImageMap *test_mock_image_map; + + MockListener(TestMockImageMap *test_mock_image_map) + : test_mock_image_map(test_mock_image_map) { + } + + MOCK_METHOD2(mock_acquire_image, void(const std::string &, Context*)); + MOCK_METHOD2(mock_release_image, void(const std::string &, Context*)); + MOCK_METHOD3(mock_remove_image, void(const std::string &, + const std::string &, Context*)); + + void acquire_image(const std::string &global_image_id, + const std::string &instance_id, Context* on_finish) { + mock_acquire_image(global_image_id, on_finish); + } + + void release_image(const std::string &global_image_id, + const std::string &instance_id, Context* on_finish) { + mock_release_image(global_image_id, on_finish); + } + + void remove_image(const std::string &mirror_uuid, + const std::string &global_image_id, + const std::string &instance_id, Context* on_finish) { + mock_remove_image(mirror_uuid, global_image_id, on_finish); + } + }; + + TestMockImageMap() + : m_lock("TestMockImageMap::m_lock"), + m_notify_update_count(0), + m_map_update_count(0) { + } + + void SetUp() override { + TestFixture::SetUp(); + + m_local_instance_id = stringify(m_local_io_ctx.get_instance_id()); + + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", + "0")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "simple")); + } + + void TearDown() override { + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_type", "none")); + + TestFixture::TearDown(); + } + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_,_)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = new FunctionContext([this, ctx](int r) { + Mutex::Locker timer_locker(m_threads->timer_lock); + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), ReturnArg<1>())); + } + + void expect_rebalance_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_,_)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + // disable rebalance so as to not reschedule it again + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "0"); + + auto wrapped_ctx = new FunctionContext([this, ctx](int r) { + Mutex::Locker timer_locker(m_threads->timer_lock); + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), ReturnArg<1>())); + } + + void expect_load_request(MockLoadRequest &request, int r) { + EXPECT_CALL(request, send()) + .WillOnce(Invoke([&request, r]() { + request.on_finish->complete(r); + })); + } + + void expect_update_request(MockUpdateRequest &request, int r) { + EXPECT_CALL(request, send()) + .WillOnce(Invoke([this, &request, r]() { + request.on_finish->complete(r); + if (r == 0) { + Mutex::Locker locker(m_lock); + ++m_map_update_count; + m_cond.Signal(); + } + })); + } + + void expect_listener_acquire_image(MockListener &mock_listener, + const std::string &global_image_id, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_acquire_image(global_image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { + Mutex::Locker locker(m_lock); + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.Signal(); + }))); + } + + void expect_listener_release_image(MockListener &mock_listener, + const std::string &global_image_id, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_release_image(global_image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { + Mutex::Locker locker(m_lock); + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.Signal(); + }))); + } + + void expect_listener_remove_image(MockListener &mock_listener, + const std::string &mirror_uuid, + const std::string &global_image_id, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, + mock_remove_image(mirror_uuid, global_image_id, _)) + .WillOnce(WithArg<2>(Invoke([this, global_image_id, peer_ack_ctxs](Context* ctx) { + Mutex::Locker locker(m_lock); + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.Signal(); + }))); + } + + void expect_listener_images_unmapped(MockListener &mock_listener, size_t count, + std::set<std::string> *global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + EXPECT_CALL(mock_listener, mock_release_image(_, _)) + .Times(count) + .WillRepeatedly(Invoke([this, global_image_ids, peer_ack_ctxs](std::string global_image_id, Context* ctx) { + Mutex::Locker locker(m_lock); + global_image_ids->emplace(global_image_id); + peer_ack_ctxs->insert({global_image_id, ctx}); + ++m_notify_update_count; + m_cond.Signal(); + })); + } + + void remote_peer_ack_nowait(MockImageMap *image_map, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + auto ack_ctx = it->second; + peer_ack_ctxs->erase(it); + ack_ctx->complete(ret); + wait_for_scheduled_task(); + } + } + + void remote_peer_ack_wait(MockImageMap *image_map, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + auto ack_ctx = it->second; + peer_ack_ctxs->erase(it); + ack_ctx->complete(ret); + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_map_update(1)); + } + } + + void remote_peer_ack_listener_wait(MockImageMap *image_map, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto& global_image_id : global_image_ids) { + auto it = peer_ack_ctxs->find(global_image_id); + ASSERT_TRUE(it != peer_ack_ctxs->end()); + auto ack_ctx = it->second; + peer_ack_ctxs->erase(it); + ack_ctx->complete(ret); + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(1)); + } + } + + void update_map_and_acquire(MockThreads &mock_threads, + MockUpdateRequest &mock_update_request, + MockListener &mock_listener, + const std::set<std::string> &global_image_ids, + int ret, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_add_event(mock_threads); + expect_update_request(mock_update_request, ret); + expect_add_event(mock_threads); + expect_listener_acquire_image(mock_listener, global_image_id, + peer_ack_ctxs); + } + } + + void update_map_request(MockThreads &mock_threads, + MockUpdateRequest &mock_update_request, + const std::set<std::string> &global_image_ids, int ret) { + for (uint32_t i = 0; i < global_image_ids.size(); ++i) { + expect_add_event(mock_threads); + expect_update_request(mock_update_request, ret); + } + } + + void wait_for_scheduled_task() { + m_threads->work_queue->drain(); + } + + bool wait_for_listener_notify(uint32_t count) { + Mutex::Locker locker(m_lock); + while (m_notify_update_count < count) { + if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) { + break; + } + } + + if (m_notify_update_count < count) { + return false; + } + + m_notify_update_count -= count; + return true; + } + + bool wait_for_map_update(uint32_t count) { + Mutex::Locker locker(m_lock); + while (m_map_update_count < count) { + if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) { + break; + } + } + + if (m_map_update_count < count) { + return false; + } + + m_map_update_count -= count; + return true; + } + + int when_shut_down(MockImageMap *image_map) { + C_SaferCond ctx; + image_map->shut_down(&ctx); + return ctx.wait(); + } + + void listener_acquire_images(MockListener &mock_listener, + const std::set<std::string> &global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_listener_acquire_image(mock_listener, global_image_id, + peer_ack_ctxs); + } + } + + void listener_release_images(MockListener &mock_listener, + const std::set<std::string> &global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_listener_release_image(mock_listener, global_image_id, + peer_ack_ctxs); + } + } + + void listener_remove_images(MockListener &mock_listener, + const std::string &mirror_uuid, + std::set<std::string> &global_image_ids, + std::map<std::string, Context*> *peer_ack_ctxs) { + for (auto const &global_image_id : global_image_ids) { + expect_listener_remove_image(mock_listener, mirror_uuid, global_image_id, + peer_ack_ctxs); + } + } + + Mutex m_lock; + Cond m_cond; + uint32_t m_notify_update_count; + uint32_t m_map_update_count; + std::string m_local_instance_id; +}; + +TEST_F(TestMockImageMap, SetLocalImages) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddRemoveLocalImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("", std::move(initial_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // RELEASE+REMOVE_MAPPING + expect_add_event(mock_threads); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, + 0); + + // remove images + mock_image_map->update_images("", {}, std::move(remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size())); + + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddRemoveRemoteImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), + {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // RELEASE+REMOVE_MAPPING + std::map<std::string, Context*> peer_remove_ack_ctxs; + listener_remove_images(mock_listener, "uuid1", remove_global_image_ids, + &peer_remove_ack_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, + 0); + + // remove images + mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_remove_ack_ctxs); + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddRemoveRemoteImageDuplicateNotification) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_dup(initial_global_image_ids); + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> remove_global_image_ids_dup(remove_global_image_ids); + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // trigger duplicate "add" event + wait_for_scheduled_task(); + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids_dup), {}); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // RELEASE+REMOVE_MAPPING + std::map<std::string, Context*> peer_remove_ack_ctxs; + listener_remove_images(mock_listener, "uuid1", remove_global_image_ids, + &peer_remove_ack_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remove_global_image_ids, 0); + + // remove images + mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remove_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_remove_ack_ctxs); + remote_peer_ack_wait(mock_image_map.get(), remove_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // trigger duplicate "remove" notification + mock_image_map->update_images("uuid1", {}, std::move(remove_global_image_ids_dup)); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AcquireImageErrorRetry) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> initial_global_image_ids{ + "global id 1", "global id 2" + }; + std::set<std::string> initial_global_image_ids_ack(initial_global_image_ids); + + // UPDATE_MAPPING failure + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, -EIO); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(initial_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), initial_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RemoveRemoteAndLocalImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + // remote image set + std::set<std::string> initial_remote_global_image_ids{ + "global id 1" + }; + std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids); + + // local image set + std::set<std::string> initial_local_global_image_ids{ + "global id 1" + }; + + // remote/local images to remove + std::set<std::string> remote_remove_global_image_ids{ + "global id 1" + }; + std::set<std::string> remote_remove_global_image_ids_ack(remote_remove_global_image_ids); + + std::set<std::string> local_remove_global_image_ids{ + "global id 1" + }; + std::set<std::string> local_remove_global_image_ids_ack(local_remove_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_remote_global_image_ids, + &peer_ack_ctxs); + + // initial remote image list + mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), + initial_remote_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // set initial local image list -- this is a no-op from policy pov + mock_image_map->update_images("", std::move(initial_local_global_image_ids), {}); + + // remove remote images -- this should be a no-op from policy pov + // except the listener notification + std::map<std::string, Context*> peer_ack_remove_ctxs; + listener_remove_images(mock_listener, "uuid1", remote_remove_global_image_ids, + &peer_ack_remove_ctxs); + + mock_image_map->update_images("uuid1", {}, std::move(remote_remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remote_remove_global_image_ids_ack.size())); + + // RELEASE+REMOVE_MAPPING + expect_add_event(mock_threads); + listener_release_images(mock_listener, local_remove_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, local_remove_global_image_ids, 0); + + // remove local images + mock_image_map->update_images("", {}, std::move(local_remove_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(local_remove_global_image_ids_ack.size())); + + remote_peer_ack_nowait(mock_image_map.get(), local_remove_global_image_ids_ack, + 0, &peer_ack_remove_ctxs); + remote_peer_ack_wait(mock_image_map.get(), local_remove_global_image_ids_ack, + 0, &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddInstance) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5" + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RemoveInstance) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5" + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + expect_add_event(mock_threads); + + // UPDATE_MAPPING+ACQUIRE + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // set initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request -- completing action + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + shuffled_global_image_ids.clear(); + + // remove added instance + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_removed({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddInstancePingPongImageTest) { + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_image_policy_migration_throttle", "600")); + + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6", "global id 7", "global id 8", "global id 9", "global id 10", + "global id 11", "global id 12", "global id 13", "global id 14" + }; + + std::map<std::string, cls::rbd::MirrorImageMap> image_mapping; + for (auto& global_image_id : global_image_ids) { + image_mapping[global_image_id] = {m_local_instance_id, {}, {}}; + } + + // ACQUIRE + MockLoadRequest mock_load_request; + EXPECT_CALL(mock_load_request, send()).WillOnce( + Invoke([&mock_load_request, &image_mapping]() { + *mock_load_request.image_map = image_mapping; + mock_load_request.on_finish->complete(0); + })); + + expect_add_event(mock_threads); + MockListener mock_listener(this); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> global_image_ids_ack(global_image_ids); + + // remote peer ACKs image acquire request -- completing action + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // set initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request -- completing action + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 7, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> migrated_global_image_ids(shuffled_global_image_ids); + shuffled_global_image_ids.clear(); + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 3, &shuffled_global_image_ids, + &peer_ack_ctxs); + + // add another instance + mock_image_map->update_instances_added({"5432"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + // shuffle set should be distinct + std::set<std::string> reshuffled; + std::set_intersection(migrated_global_image_ids.begin(), migrated_global_image_ids.end(), + shuffled_global_image_ids.begin(), shuffled_global_image_ids.end(), + std::inserter(reshuffled, reshuffled.begin())); + ASSERT_TRUE(reshuffled.empty()); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RemoveInstanceWithRemoveImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "remote id 4", + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + std::set<std::string> remove_global_image_ids{ + "global id 1" + }; + std::set<std::string> remove_global_image_ids_ack(remove_global_image_ids); + + expect_add_event(mock_threads); + // UPDATE_MAPPING+ACQUIRE + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids); + + // RELEASE + + std::map<std::string, Context*> peer_ack_remove_ctxs; + listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids, + &peer_ack_remove_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + + mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2)); + + // instance failed -- update policy for instance removal + mock_image_map->update_instances_removed({"9876"}); + + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, + -ENOENT, &peer_ack_remove_ctxs); + remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids, + -EBLACKLISTED, &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, AddErrorAndRemoveImage) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "remote id 4", + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("uuid1", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_removed({"9876"}); + + std::set<std::string> released_global_image_ids; + std::map<std::string, Context*> release_peer_ack_ctxs; + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids, + &release_peer_ack_ctxs); + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 1, &released_global_image_ids, + &release_peer_ack_ctxs); + + // instance blacklisted -- ACQUIRE request fails + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, + -EBLACKLISTED, &peer_ack_ctxs); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + std::map<std::string, Context*> remap_peer_ack_ctxs; + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &remap_peer_ack_ctxs); + + // instance blacklisted -- RELEASE request fails + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + -ENOENT, &release_peer_ack_ctxs); + wait_for_scheduled_task(); + + // new peer acks acquire request + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &remap_peer_ack_ctxs); + wait_for_scheduled_task(); + + std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids); + + // remove image + std::map<std::string, Context*> peer_ack_remove_ctxs; + listener_remove_images(mock_listener, "uuid1", shuffled_global_image_ids, + &peer_ack_remove_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, 0); + + mock_image_map->update_images("uuid1", {}, std::move(shuffled_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids_ack, 0, + &peer_ack_remove_ctxs); + remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, MirrorUUIDUpdated) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + // remote image set + std::set<std::string> initial_remote_global_image_ids{ + "global id 1", "global id 2", "global id 3" + }; + std::set<std::string> initial_remote_global_image_ids_ack(initial_remote_global_image_ids); + + // remote/local images to remove + std::set<std::string> remote_removed_global_image_ids{ + "global id 1", "global id 2", "global id 3" + }; + std::set<std::string> remote_removed_global_image_ids_ack(remote_removed_global_image_ids); + + std::set<std::string> remote_added_global_image_ids{ + "global id 1", "global id 2", "global id 3" + }; + std::set<std::string> remote_added_global_image_ids_ack(remote_added_global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, initial_remote_global_image_ids, + &peer_ack_ctxs); + + // initial remote image list + mock_image_map->update_images("uuid1", std::move(initial_remote_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(initial_remote_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), + initial_remote_global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + // RELEASE+REMOVE_MAPPING + std::map<std::string, Context*> peer_remove_ack_ctxs; + listener_remove_images(mock_listener, "uuid1", remote_removed_global_image_ids, + &peer_remove_ack_ctxs); + expect_add_event(mock_threads); + listener_release_images(mock_listener, remote_removed_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, remote_removed_global_image_ids, 0); + + mock_image_map->update_images("uuid1", {}, std::move(remote_removed_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(remote_removed_global_image_ids_ack.size() * 2)); + + remote_peer_ack_nowait(mock_image_map.get(), + remote_removed_global_image_ids_ack, 0, + &peer_remove_ack_ctxs); + remote_peer_ack_wait(mock_image_map.get(), + remote_removed_global_image_ids_ack, 0, + &peer_ack_ctxs); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + listener_acquire_images(mock_listener, remote_added_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_images("uuid2", std::move(remote_added_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(remote_added_global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), + remote_added_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +TEST_F(TestMockImageMap, RebalanceImageMap) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + + MockLoadRequest mock_load_request; + expect_load_request(mock_load_request, 0); + + MockListener mock_listener(this); + + std::unique_ptr<MockImageMap> mock_image_map{ + MockImageMap::create(m_local_io_ctx, &mock_threads, m_local_instance_id, + mock_listener)}; + + C_SaferCond cond; + mock_image_map->init(&cond); + ASSERT_EQ(0, cond.wait()); + + std::set<std::string> global_image_ids{ + "global id 1", "global id 2", "global id 3", "global id 4", "global id 5", + "global id 6", "global id 7", "global id 8", "global id 9", "global id 10", + }; + std::set<std::string> global_image_ids_ack(global_image_ids); + + // UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + MockUpdateRequest mock_update_request; + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + std::map<std::string, Context*> peer_ack_ctxs; + listener_acquire_images(mock_listener, global_image_ids, + &peer_ack_ctxs); + + // initial image list + mock_image_map->update_images("", std::move(global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(global_image_ids_ack.size())); + + // remote peer ACKs image acquire request + remote_peer_ack_nowait(mock_image_map.get(), global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + mock_image_map->update_instances_added({m_local_instance_id}); + + std::set<std::string> shuffled_global_image_ids; + + // RELEASE+UPDATE_MAPPING+ACQUIRE + expect_add_event(mock_threads); + expect_listener_images_unmapped(mock_listener, 5, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_instances_added({"9876"}); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + + // remove all shuffled images -- make way for rebalance + std::set<std::string> shuffled_global_image_ids_ack(shuffled_global_image_ids); + + // RELEASE+REMOVE_MAPPING + expect_add_event(mock_threads); + listener_release_images(mock_listener, shuffled_global_image_ids, + &peer_ack_ctxs); + update_map_request(mock_threads, mock_update_request, shuffled_global_image_ids, + 0); + + mock_image_map->update_images("", {}, std::move(shuffled_global_image_ids)); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids_ack.size())); + + remote_peer_ack_wait(mock_image_map.get(), shuffled_global_image_ids_ack, 0, + &peer_ack_ctxs); + wait_for_scheduled_task(); + + shuffled_global_image_ids.clear(); + shuffled_global_image_ids_ack.clear(); + + std::set<std::string> new_global_image_ids = { + "global id 11" + }; + std::set<std::string> new_global_image_ids_ack(new_global_image_ids); + + expect_add_event(mock_threads); + expect_update_request(mock_update_request, 0); + expect_add_event(mock_threads); + listener_acquire_images(mock_listener, new_global_image_ids, &peer_ack_ctxs); + + expect_rebalance_event(mock_threads); // rebalance task + expect_add_event(mock_threads); // update task scheduled by + // rebalance task + expect_listener_images_unmapped(mock_listener, 2, &shuffled_global_image_ids, + &peer_ack_ctxs); + + mock_image_map->update_images("", std::move(new_global_image_ids), {}); + + ASSERT_TRUE(wait_for_map_update(1)); + ASSERT_TRUE(wait_for_listener_notify(new_global_image_ids_ack.size())); + + // set rebalance interval + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + cct->_conf.set_val("rbd_mirror_image_policy_rebalance_timeout", "5"); + remote_peer_ack_nowait(mock_image_map.get(), new_global_image_ids_ack, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_TRUE(wait_for_listener_notify(shuffled_global_image_ids.size())); + + update_map_and_acquire(mock_threads, mock_update_request, + mock_listener, shuffled_global_image_ids, 0, + &peer_ack_ctxs); + remote_peer_ack_listener_wait(mock_image_map.get(), shuffled_global_image_ids, + 0, &peer_ack_ctxs); + + // completion shuffle action for now (re)mapped images + remote_peer_ack_nowait(mock_image_map.get(), shuffled_global_image_ids, 0, + &peer_ack_ctxs); + + wait_for_scheduled_task(); + ASSERT_EQ(0, when_shut_down(mock_image_map.get())); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc new file mode 100644 index 00000000..1bd330fd --- /dev/null +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -0,0 +1,1397 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Replay.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/BootstrapRequest.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/EventPreprocessor.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockJournal.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" + +namespace librbd { + +namespace { + +struct MockTestJournal; + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } + MockTestJournal *journal = nullptr; +}; + +struct MockTestJournal : public MockJournal { + MOCK_METHOD2(start_external_replay, void(journal::Replay<MockTestImageCtx> **, + Context *on_start)); + MOCK_METHOD0(stop_external_replay, void()); +}; + +} // anonymous namespace + +namespace journal { + +template<> +struct Replay<MockTestImageCtx> { + MOCK_METHOD2(decode, int(bufferlist::const_iterator *, EventEntry *)); + MOCK_METHOD3(process, void(const EventEntry &, Context *, Context *)); + MOCK_METHOD1(flush, void(Context*)); + MOCK_METHOD2(shut_down, void(bool, Context*)); +}; + +template <> +struct TypeTraits<MockTestImageCtx> { + typedef ::journal::MockJournalerProxy Journaler; + typedef ::journal::MockReplayEntryProxy ReplayEntry; +}; + +struct MirrorPeerClientMeta; + +} // namespace journal +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct ImageDeleter<librbd::MockTestImageCtx> { + static ImageDeleter* s_instance; + + static void trash_move(librados::IoCtx& local_io_ctx, + const std::string& global_image_id, bool resync, + MockContextWQ* work_queue, Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->trash_move(global_image_id, resync, on_finish); + } + + MOCK_METHOD3(trash_move, void(const std::string&, bool, Context*)); + + ImageDeleter() { + s_instance = this; + } +}; + +ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr; + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + Mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +template<> +class InstanceWatcher<librbd::MockTestImageCtx> { +}; + +namespace image_replayer { + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::MatcherCast; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::SetArgPointee; +using ::testing::WithArg; + +template<> +struct PrepareLocalImageRequest<librbd::MockTestImageCtx> { + static PrepareLocalImageRequest* s_instance; + std::string *local_image_id = nullptr; + std::string *local_image_name = nullptr; + std::string *tag_owner = nullptr; + Context *on_finish = nullptr; + + static PrepareLocalImageRequest* create(librados::IoCtx &, + const std::string &global_image_id, + std::string *local_image_id, + std::string *local_image_name, + std::string *tag_owner, + MockContextWQ *work_queue, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->local_image_id = local_image_id; + s_instance->local_image_name = local_image_name; + s_instance->tag_owner = tag_owner; + s_instance->on_finish = on_finish; + return s_instance; + } + + PrepareLocalImageRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct PrepareRemoteImageRequest<librbd::MockTestImageCtx> { + static PrepareRemoteImageRequest* s_instance; + std::string *remote_mirror_uuid = nullptr; + std::string *remote_image_id = nullptr; + cls::journal::ClientState *client_state; + ::journal::MockJournalerProxy **remote_journaler = nullptr; + librbd::journal::MirrorPeerClientMeta *client_meta = nullptr; + Context *on_finish = nullptr; + + static PrepareRemoteImageRequest* create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &local_image_id, + const journal::Settings &settings, + std::string *remote_mirror_uuid, + std::string *remote_image_id, + ::journal::MockJournalerProxy **remote_journaler, + cls::journal::ClientState *client_state, + librbd::journal::MirrorPeerClientMeta *client_meta, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->remote_mirror_uuid = remote_mirror_uuid; + s_instance->remote_image_id = remote_image_id; + s_instance->remote_journaler = remote_journaler; + s_instance->client_state = client_state; + s_instance->client_meta = client_meta; + s_instance->on_finish = on_finish; + return s_instance; + } + + PrepareRemoteImageRequest() { + s_instance = this; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct BootstrapRequest<librbd::MockTestImageCtx> { + static BootstrapRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + bool *do_resync = nullptr; + + static BootstrapRequest* create( + Threads<librbd::MockTestImageCtx>* threads, + librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, + rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher, + librbd::MockTestImageCtx **local_image_ctx, + const std::string &local_image_name, const std::string &remote_image_id, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const std::string &remote_mirror_uuid, + ::journal::MockJournalerProxy *journaler, + cls::journal::ClientState *client_state, + librbd::journal::MirrorPeerClientMeta *client_meta, + Context *on_finish, bool *do_resync, + rbd::mirror::ProgressContext *progress_ctx = nullptr) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = local_image_ctx; + s_instance->on_finish = on_finish; + s_instance->do_resync = do_resync; + return s_instance; + } + + BootstrapRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~BootstrapRequest() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + void put() { + } + + void get() { + } + + inline bool is_syncing() const { + return false; + } + + MOCK_METHOD0(send, void()); + MOCK_METHOD0(cancel, void()); +}; + +template<> +struct CloseImageRequest<librbd::MockTestImageCtx> { + static CloseImageRequest* s_instance; + librbd::MockTestImageCtx **image_ctx = nullptr; + Context *on_finish = nullptr; + + static CloseImageRequest* create(librbd::MockTestImageCtx **image_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ctx = image_ctx; + s_instance->on_finish = on_finish; + return s_instance; + } + + CloseImageRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~CloseImageRequest() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +template<> +struct EventPreprocessor<librbd::MockTestImageCtx> { + static EventPreprocessor *s_instance; + + static EventPreprocessor *create(librbd::MockTestImageCtx &local_image_ctx, + ::journal::MockJournalerProxy &remote_journaler, + const std::string &local_mirror_uuid, + librbd::journal::MirrorPeerClientMeta *client_meta, + MockContextWQ *work_queue) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + static void destroy(EventPreprocessor* processor) { + } + + EventPreprocessor() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~EventPreprocessor() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD1(is_required, bool(const librbd::journal::EventEntry &)); + MOCK_METHOD2(preprocess, void(librbd::journal::EventEntry *, Context *)); +}; + +template<> +struct ReplayStatusFormatter<librbd::MockTestImageCtx> { + static ReplayStatusFormatter* s_instance; + + static ReplayStatusFormatter* create(::journal::MockJournalerProxy *journaler, + const std::string &mirror_uuid) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + static void destroy(ReplayStatusFormatter* formatter) { + } + + ReplayStatusFormatter() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~ReplayStatusFormatter() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD2(get_or_send_update, bool(std::string *description, Context *on_finish)); +}; + +BootstrapRequest<librbd::MockTestImageCtx>* BootstrapRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +CloseImageRequest<librbd::MockTestImageCtx>* CloseImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +EventPreprocessor<librbd::MockTestImageCtx>* EventPreprocessor<librbd::MockTestImageCtx>::s_instance = nullptr; +PrepareLocalImageRequest<librbd::MockTestImageCtx>* PrepareLocalImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +PrepareRemoteImageRequest<librbd::MockTestImageCtx>* PrepareRemoteImageRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +ReplayStatusFormatter<librbd::MockTestImageCtx>* ReplayStatusFormatter<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/ImageReplayer.cc" + +namespace rbd { +namespace mirror { + +class TestMockImageReplayer : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef ImageDeleter<librbd::MockTestImageCtx> MockImageDeleter; + typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest; + typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest; + typedef EventPreprocessor<librbd::MockTestImageCtx> MockEventPreprocessor; + typedef PrepareLocalImageRequest<librbd::MockTestImageCtx> MockPrepareLocalImageRequest; + typedef PrepareRemoteImageRequest<librbd::MockTestImageCtx> MockPrepareRemoteImageRequest; + typedef ReplayStatusFormatter<librbd::MockTestImageCtx> MockReplayStatusFormatter; + typedef librbd::journal::Replay<librbd::MockTestImageCtx> MockReplay; + typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + } + + void TearDown() override { + delete m_image_replayer; + + TestMockFixture::TearDown(); + } + + void create_local_image() { + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_work_queue_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event_after_repeatedly(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillRepeatedly( + DoAll(Invoke([this](double seconds, Context *ctx) { + m_threads->timer->add_event_after(seconds, ctx); + }), + ReturnArg<1>())); + EXPECT_CALL(*mock_threads.timer, cancel_event(_)) + .WillRepeatedly( + Invoke([this](Context *ctx) { + return m_threads->timer->cancel_event(ctx); + })); + } + + void expect_flush_repeatedly(MockReplay& mock_replay, + journal::MockJournaler& mock_journal) { + EXPECT_CALL(mock_replay, flush(_)) + .WillRepeatedly(Invoke([this](Context* ctx) { + m_threads->work_queue->queue(ctx, 0); + })); + EXPECT_CALL(mock_journal, flush_commit_position(_)) + .WillRepeatedly(Invoke([this](Context* ctx) { + m_threads->work_queue->queue(ctx, 0); + })); + } + + void expect_trash_move(MockImageDeleter& mock_image_deleter, + const std::string& global_image_id, + bool ignore_orphan, int r) { + EXPECT_CALL(mock_image_deleter, + trash_move(global_image_id, ignore_orphan, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { + bufferlist bl; + encode(tag_data, bl); + return bl; + } + + void expect_get_or_send_update( + MockReplayStatusFormatter &mock_replay_status_formatter) { + EXPECT_CALL(mock_replay_status_formatter, get_or_send_update(_, _)) + .WillRepeatedly(DoAll(WithArg<1>(CompleteContext(-EEXIST)), + Return(true))); + } + + void expect_send(MockPrepareLocalImageRequest &mock_request, + const std::string &local_image_id, + const std::string &local_image_name, + const std::string &tag_owner, + int r) { + EXPECT_CALL(mock_request, send()) + .WillOnce(Invoke([&mock_request, local_image_id, local_image_name, tag_owner, r]() { + if (r == 0) { + *mock_request.local_image_id = local_image_id; + *mock_request.local_image_name = local_image_name; + *mock_request.tag_owner = tag_owner; + } + mock_request.on_finish->complete(r); + })); + } + + void expect_send(MockPrepareRemoteImageRequest& mock_request, + const std::string& mirror_uuid, const std::string& image_id, + int r) { + EXPECT_CALL(mock_request, send()) + .WillOnce(Invoke([&mock_request, image_id, mirror_uuid, r]() { + if (r >= 0) { + *mock_request.remote_journaler = new ::journal::MockJournalerProxy(); + } + + *mock_request.remote_mirror_uuid = mirror_uuid; + *mock_request.remote_image_id = image_id; + mock_request.on_finish->complete(r); + })); + } + + void expect_send(MockBootstrapRequest &mock_bootstrap_request, + librbd::MockTestImageCtx &mock_local_image_ctx, + bool do_resync, int r) { + EXPECT_CALL(mock_bootstrap_request, send()) + .WillOnce(Invoke([&mock_bootstrap_request, &mock_local_image_ctx, + do_resync, r]() { + if (r == 0) { + *mock_bootstrap_request.image_ctx = &mock_local_image_ctx; + *mock_bootstrap_request.do_resync = do_resync; + } + mock_bootstrap_request.on_finish->complete(r); + })); + } + + void expect_start_external_replay(librbd::MockTestJournal &mock_journal, + MockReplay *mock_replay, int r) { + EXPECT_CALL(mock_journal, start_external_replay(_, _)) + .WillOnce(DoAll(SetArgPointee<0>(mock_replay), + WithArg<1>(CompleteContext(r)))); + } + + void expect_init(::journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, init(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_get_cached_client(::journal::MockJournaler &mock_journaler, + int r) { + librbd::journal::ImageClientMeta image_client_meta; + image_client_meta.tag_class = 0; + + librbd::journal::ClientData client_data; + client_data.client_meta = image_client_meta; + + cls::journal::Client client; + encode(client_data, client.data); + + EXPECT_CALL(mock_journaler, get_cached_client("local_mirror_uuid", _)) + .WillOnce(DoAll(SetArgPointee<1>(client), + Return(r))); + } + + void expect_stop_replay(::journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, stop_replay(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_flush(MockReplay &mock_replay, int r) { + EXPECT_CALL(mock_replay, flush(_)).WillOnce(CompleteContext(r)); + } + + void expect_shut_down(MockReplay &mock_replay, bool cancel_ops, int r) { + EXPECT_CALL(mock_replay, shut_down(cancel_ops, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + + void expect_shut_down(journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, shut_down(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_send(MockCloseImageRequest &mock_close_image_request, int r) { + EXPECT_CALL(mock_close_image_request, send()) + .WillOnce(Invoke([&mock_close_image_request, r]() { + *mock_close_image_request.image_ctx = nullptr; + mock_close_image_request.on_finish->complete(r); + })); + } + + void expect_get_commit_tid_in_debug( + ::journal::MockReplayEntry &mock_replay_entry) { + // It is used in debug messages and depends on debug level + EXPECT_CALL(mock_replay_entry, get_commit_tid()) + .Times(AtLeast(0)) + .WillRepeatedly(Return(0)); + } + + void expect_get_tag_tid_in_debug(librbd::MockTestJournal &mock_journal) { + // It is used in debug messages and depends on debug level + EXPECT_CALL(mock_journal, get_tag_tid()).Times(AtLeast(0)) + .WillRepeatedly(Return(0)); + } + + void expect_committed(::journal::MockReplayEntry &mock_replay_entry, + ::journal::MockJournaler &mock_journaler, int times) { + EXPECT_CALL(mock_replay_entry, get_data()).Times(times); + EXPECT_CALL(mock_journaler, committed( + MatcherCast<const ::journal::MockReplayEntryProxy&>(_))) + .Times(times); + } + + void expect_try_pop_front(::journal::MockJournaler &mock_journaler, + uint64_t replay_tag_tid, bool entries_available) { + EXPECT_CALL(mock_journaler, try_pop_front(_, _)) + .WillOnce(DoAll(SetArgPointee<0>(::journal::MockReplayEntryProxy()), + SetArgPointee<1>(replay_tag_tid), + Return(entries_available))); + } + + void expect_try_pop_front_return_no_entries( + ::journal::MockJournaler &mock_journaler, Context *on_finish) { + EXPECT_CALL(mock_journaler, try_pop_front(_, _)) + .WillOnce(DoAll(Invoke([on_finish](::journal::MockReplayEntryProxy *e, + uint64_t *t) { + on_finish->complete(0); + }), + Return(false))); + } + + void expect_get_tag(::journal::MockJournaler &mock_journaler, + const cls::journal::Tag &tag, int r) { + EXPECT_CALL(mock_journaler, get_tag(_, _, _)) + .WillOnce(DoAll(SetArgPointee<1>(tag), + WithArg<2>(CompleteContext(r)))); + } + + void expect_allocate_tag(librbd::MockTestJournal &mock_journal, int r) { + EXPECT_CALL(mock_journal, allocate_tag(_, _, _)) + .WillOnce(WithArg<2>(CompleteContext(r))); + } + + void expect_preprocess(MockEventPreprocessor &mock_event_preprocessor, + bool required, int r) { + EXPECT_CALL(mock_event_preprocessor, is_required(_)) + .WillOnce(Return(required)); + if (required) { + EXPECT_CALL(mock_event_preprocessor, preprocess(_, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + } + + void expect_process(MockReplay &mock_replay, + int on_ready_r, int on_commit_r) { + EXPECT_CALL(mock_replay, process(_, _, _)) + .WillOnce(DoAll(WithArg<1>(CompleteContext(on_ready_r)), + WithArg<2>(CompleteContext(on_commit_r)))); + } + + void create_image_replayer(MockThreads &mock_threads) { + m_image_replayer = new MockImageReplayer( + &mock_threads, &m_instance_watcher, + rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)), + "local_mirror_uuid", m_local_io_ctx.get_id(), "global image id"); + m_image_replayer->add_peer("peer_uuid", m_remote_io_ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx = nullptr; + MockInstanceWatcher m_instance_watcher; + MockImageReplayer *m_image_replayer = nullptr; +}; + +TEST_F(TestMockImageReplayer, StartStop) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + librbd::MockTestJournal mock_local_journal; + mock_local_image_ctx.journal = &mock_local_journal; + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplay mock_local_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_flush_repeatedly(mock_local_replay, mock_remote_journaler); + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); + + EXPECT_CALL(mock_local_journal, add_listener(_)); + + expect_init(mock_remote_journaler, 0); + + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, 0); + + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + + EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _)); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + ASSERT_EQ(image_replayer::HEALTH_STATE_OK, + m_image_replayer->get_health_state()); + + // STOP + + MockCloseImageRequest mock_close_local_image_request; + + expect_shut_down(mock_local_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_send(mock_close_local_image_request, 0); + + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(0, stop_ctx.wait()); + ASSERT_EQ(image_replayer::HEALTH_STATE_OK, + m_image_replayer->get_health_state()); +} + +TEST_F(TestMockImageReplayer, LocalImagePrimary) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + "remote image id", 0); + EXPECT_CALL(mock_remote_journaler, construct()); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, LocalImageDNE) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, "", "", "", -ENOENT); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EREMOTEIO); + + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EREMOTEIO, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, PrepareLocalImageError) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", -EINVAL); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EINVAL, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, GetRemoteImageIdDNE) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + "", -ENOENT); + expect_trash_move(mock_image_deleter, "global image id", false, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, GetRemoteImageIdNonLinkedDNE) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "some other mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + "", -ENOENT); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-ENOENT, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, GetRemoteImageIdError) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, -EINVAL); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EINVAL, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, BootstrapError) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, -EINVAL); + + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EINVAL, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, StopBeforeBootstrap) { + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()) + .WillOnce(Invoke([this]() { + m_image_replayer->stop(nullptr, true); + })); + + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-ECANCELED, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, StartExternalReplayError) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + librbd::MockTestJournal mock_local_journal; + mock_local_image_ctx.journal = &mock_local_journal; + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplay mock_local_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); + + EXPECT_CALL(mock_local_journal, add_listener(_)); + + expect_init(mock_remote_journaler, 0); + + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, 0); + + expect_start_external_replay(mock_local_journal, nullptr, -EINVAL); + + MockCloseImageRequest mock_close_local_image_request; + EXPECT_CALL(mock_local_journal, remove_listener(_)); + expect_send(mock_close_local_image_request, 0); + + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(-EINVAL, start_ctx.wait()); + ASSERT_EQ(image_replayer::HEALTH_STATE_ERROR, + m_image_replayer->get_health_state()); +} + +TEST_F(TestMockImageReplayer, StopError) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + librbd::MockTestJournal mock_local_journal; + mock_local_image_ctx.journal = &mock_local_journal; + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplay mock_local_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + + expect_flush_repeatedly(mock_local_replay, mock_remote_journaler); + expect_get_or_send_update(mock_replay_status_formatter); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); + + EXPECT_CALL(mock_local_journal, add_listener(_)); + + expect_init(mock_remote_journaler, 0); + + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, 0); + + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + + EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _)); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // STOP (errors are ignored) + + MockCloseImageRequest mock_close_local_image_request; + + expect_shut_down(mock_local_replay, true, -EINVAL); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_send(mock_close_local_image_request, -EINVAL); + + expect_stop_replay(mock_remote_journaler, -EINVAL); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, -EINVAL); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(0, stop_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, Replay) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + librbd::MockTestJournal mock_local_journal; + mock_local_image_ctx.journal = &mock_local_journal; + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplay mock_local_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + ::journal::MockReplayEntry mock_replay_entry; + + expect_flush_repeatedly(mock_local_replay, mock_remote_journaler); + expect_get_or_send_update(mock_replay_status_formatter); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + expect_committed(mock_replay_entry, mock_remote_journaler, 2); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); + + EXPECT_CALL(mock_local_journal, add_listener(_)); + + expect_init(mock_remote_journaler, 0); + + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, 0); + + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + + EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _)); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // REPLAY + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + + // replay_flush + expect_shut_down(mock_local_replay, false, 0); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + + // process + EXPECT_CALL(mock_replay_entry, get_data()); + EXPECT_CALL(mock_local_replay, decode(_, _)) + .WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, false, 0); + expect_process(mock_local_replay, 0, 0); + + // the next event with preprocess + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + EXPECT_CALL(mock_replay_entry, get_data()); + EXPECT_CALL(mock_local_replay, decode(_, _)) + .WillOnce(Return(0)); + expect_preprocess(mock_event_preprocessor, true, 0); + expect_process(mock_local_replay, 0, 0); + + // attempt to process the next event + C_SaferCond replay_ctx; + expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx); + + // fire + m_image_replayer->handle_replay_ready(); + ASSERT_EQ(0, replay_ctx.wait()); + + // STOP + + MockCloseImageRequest mock_close_local_image_request; + expect_shut_down(mock_local_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_send(mock_close_local_image_request, 0); + + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(0, stop_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, DecodeError) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + librbd::MockTestJournal mock_local_journal; + mock_local_image_ctx.journal = &mock_local_journal; + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplay mock_local_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + ::journal::MockReplayEntry mock_replay_entry; + + expect_flush_repeatedly(mock_local_replay, mock_remote_journaler); + expect_get_or_send_update(mock_replay_status_formatter); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); + + EXPECT_CALL(mock_local_journal, add_listener(_)); + + expect_init(mock_remote_journaler, 0); + + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, 0); + + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + + EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _)); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // REPLAY + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + + // replay_flush + expect_shut_down(mock_local_replay, false, 0); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + + // process + EXPECT_CALL(mock_replay_entry, get_data()); + EXPECT_CALL(mock_local_replay, decode(_, _)) + .WillOnce(Return(-EINVAL)); + + // stop on error + expect_shut_down(mock_local_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + + MockCloseImageRequest mock_close_local_image_request; + C_SaferCond close_ctx; + EXPECT_CALL(mock_close_local_image_request, send()) + .WillOnce(Invoke([&mock_close_local_image_request, &close_ctx]() { + *mock_close_local_image_request.image_ctx = nullptr; + mock_close_local_image_request.on_finish->complete(0); + close_ctx.complete(0); + })); + + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + // fire + m_image_replayer->handle_replay_ready(); + ASSERT_EQ(0, close_ctx.wait()); + + while (!m_image_replayer->is_stopped()) { + usleep(1000); + } +} + +TEST_F(TestMockImageReplayer, DelayedReplay) { + + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + librbd::MockTestJournal mock_local_journal; + mock_local_image_ctx.journal = &mock_local_journal; + + journal::MockJournaler mock_remote_journaler; + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockPrepareLocalImageRequest mock_prepare_local_image_request; + MockPrepareRemoteImageRequest mock_prepare_remote_image_request; + MockBootstrapRequest mock_bootstrap_request; + MockReplay mock_local_replay; + MockEventPreprocessor mock_event_preprocessor; + MockReplayStatusFormatter mock_replay_status_formatter; + ::journal::MockReplayEntry mock_replay_entry; + + expect_flush_repeatedly(mock_local_replay, mock_remote_journaler); + expect_get_or_send_update(mock_replay_status_formatter); + expect_get_commit_tid_in_debug(mock_replay_entry); + expect_get_tag_tid_in_debug(mock_local_journal); + expect_committed(mock_replay_entry, mock_remote_journaler, 1); + + InSequence seq; + expect_send(mock_prepare_local_image_request, mock_local_image_ctx.id, + mock_local_image_ctx.name, "remote mirror uuid", 0); + expect_send(mock_prepare_remote_image_request, "remote mirror uuid", + m_remote_image_ctx->id, 0); + EXPECT_CALL(mock_remote_journaler, construct()); + expect_send(mock_bootstrap_request, mock_local_image_ctx, false, 0); + + EXPECT_CALL(mock_local_journal, add_listener(_)); + + expect_init(mock_remote_journaler, 0); + + EXPECT_CALL(mock_remote_journaler, add_listener(_)); + expect_get_cached_client(mock_remote_journaler, 0); + + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + + EXPECT_CALL(mock_remote_journaler, start_live_replay(_, _)); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); + + // REPLAY + + cls::journal::Tag tag = + {1, 0, encode_tag_data({librbd::Journal<>::LOCAL_MIRROR_UUID, + librbd::Journal<>::LOCAL_MIRROR_UUID, + true, 0, 0})}; + + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + + // replay_flush + expect_shut_down(mock_local_replay, false, 0); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_start_external_replay(mock_local_journal, &mock_local_replay, 0); + expect_get_tag(mock_remote_journaler, tag, 0); + expect_allocate_tag(mock_local_journal, 0); + + // process with delay + EXPECT_CALL(mock_replay_entry, get_data()); + librbd::journal::EventEntry event_entry( + librbd::journal::AioDiscardEvent(123, 345, 0), ceph_clock_now()); + EXPECT_CALL(mock_local_replay, decode(_, _)) + .WillOnce(DoAll(SetArgPointee<1>(event_entry), + Return(0))); + expect_preprocess(mock_event_preprocessor, false, 0); + expect_process(mock_local_replay, 0, 0); + + // attempt to process the next event + C_SaferCond replay_ctx; + expect_try_pop_front_return_no_entries(mock_remote_journaler, &replay_ctx); + + // fire + mock_local_image_ctx.mirroring_replay_delay = 2; + m_image_replayer->handle_replay_ready(); + ASSERT_EQ(0, replay_ctx.wait()); + + // add a pending (delayed) entry before stop + expect_try_pop_front(mock_remote_journaler, tag.tid, true); + EXPECT_CALL(mock_replay_entry, get_data()); + C_SaferCond decode_ctx; + EXPECT_CALL(mock_local_replay, decode(_, _)) + .WillOnce(DoAll(Invoke([&decode_ctx](bufferlist::const_iterator* it, + librbd::journal::EventEntry *e) { + decode_ctx.complete(0); + }), + Return(0))); + + mock_local_image_ctx.mirroring_replay_delay = 10; + m_image_replayer->handle_replay_ready(); + ASSERT_EQ(0, decode_ctx.wait()); + + // STOP + + MockCloseImageRequest mock_close_local_image_request; + + expect_shut_down(mock_local_replay, true, 0); + EXPECT_CALL(mock_local_journal, remove_listener(_)); + EXPECT_CALL(mock_local_journal, stop_external_replay()); + expect_send(mock_close_local_image_request, 0); + + expect_stop_replay(mock_remote_journaler, 0); + EXPECT_CALL(mock_remote_journaler, remove_listener(_)); + expect_shut_down(mock_remote_journaler, 0); + + C_SaferCond stop_ctx; + m_image_replayer->stop(&stop_ctx); + ASSERT_EQ(0, stop_ctx.wait()); +} + + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_ImageSync.cc b/src/test/rbd_mirror/test_mock_ImageSync.cc new file mode 100644 index 00000000..d4059140 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_ImageSync.cc @@ -0,0 +1,430 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "librbd/DeepCopyRequest.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "test/journal/mock/MockJournaler.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "tools/rbd_mirror/ImageSync.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h" +#include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + explicit MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace journal { + +template <> +struct TypeTraits<librbd::MockTestImageCtx> { + typedef ::journal::MockJournaler Journaler; +}; + +} // namespace journal + +template <> +class DeepCopyRequest<librbd::MockTestImageCtx> { +public: + static DeepCopyRequest* s_instance; + Context *on_finish; + + static DeepCopyRequest* create( + librbd::MockTestImageCtx *src_image_ctx, + librbd::MockTestImageCtx *dst_image_ctx, + librados::snap_t src_snap_id_start, librados::snap_t src_snap_id_end, + librados::snap_t dst_snap_id_start, bool flatten, + const librbd::deep_copy::ObjectNumber &object_number, + ContextWQ *work_queue, SnapSeqs *snap_seqs, ProgressContext *prog_ctx, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + DeepCopyRequest() { + s_instance = this; + } + + void put() { + } + + void get() { + } + + MOCK_METHOD0(cancel, void()); + MOCK_METHOD0(send, void()); +}; + +DeepCopyRequest<librbd::MockTestImageCtx>* DeepCopyRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +// template definitions +template class rbd::mirror::ImageSync<librbd::MockTestImageCtx>; +#include "tools/rbd_mirror/ImageSync.cc" + +namespace rbd { +namespace mirror { + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { + MOCK_METHOD2(notify_sync_request, void(const std::string, Context *)); + MOCK_METHOD1(cancel_sync_request, bool(const std::string &)); + MOCK_METHOD1(notify_sync_complete, void(const std::string &)); +}; + +namespace image_sync { + +template <> +class SyncPointCreateRequest<librbd::MockTestImageCtx> { +public: + static SyncPointCreateRequest *s_instance; + Context *on_finish; + + static SyncPointCreateRequest* create(librbd::MockTestImageCtx *remote_image_ctx, + const std::string &mirror_uuid, + journal::MockJournaler *journaler, + librbd::journal::MirrorPeerClientMeta *client_meta, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + SyncPointCreateRequest() { + s_instance = this; + } + MOCK_METHOD0(send, void()); +}; + +template <> +class SyncPointPruneRequest<librbd::MockTestImageCtx> { +public: + static SyncPointPruneRequest *s_instance; + Context *on_finish; + bool sync_complete; + + static SyncPointPruneRequest* create(librbd::MockTestImageCtx *remote_image_ctx, + bool sync_complete, + journal::MockJournaler *journaler, + librbd::journal::MirrorPeerClientMeta *client_meta, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + s_instance->sync_complete = sync_complete; + return s_instance; + } + + SyncPointPruneRequest() { + s_instance = this; + } + MOCK_METHOD0(send, void()); +}; + +SyncPointCreateRequest<librbd::MockTestImageCtx>* SyncPointCreateRequest<librbd::MockTestImageCtx>::s_instance = nullptr; +SyncPointPruneRequest<librbd::MockTestImageCtx>* SyncPointPruneRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace image_sync + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::InvokeWithoutArgs; + +class TestMockImageSync : public TestMockFixture { +public: + typedef ImageSync<librbd::MockTestImageCtx> MockImageSync; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef image_sync::SyncPointCreateRequest<librbd::MockTestImageCtx> MockSyncPointCreateRequest; + typedef image_sync::SyncPointPruneRequest<librbd::MockTestImageCtx> MockSyncPointPruneRequest; + typedef librbd::DeepCopyRequest<librbd::MockTestImageCtx> MockImageCopyRequest; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_remote_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_remote_io_ctx, m_image_name, &m_remote_image_ctx)); + + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_get_snap_id(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, get_snap_id(_, _)) + .WillOnce(Return(123)); + } + + void expect_notify_sync_request(MockInstanceWatcher &mock_instance_watcher, + const std::string &sync_id, int r) { + EXPECT_CALL(mock_instance_watcher, notify_sync_request(sync_id, _)) + .WillOnce(Invoke([this, r](const std::string &, Context *on_sync_start) { + m_threads->work_queue->queue(on_sync_start, r); + })); + } + + void expect_cancel_sync_request(MockInstanceWatcher &mock_instance_watcher, + const std::string &sync_id, bool canceled) { + EXPECT_CALL(mock_instance_watcher, cancel_sync_request(sync_id)) + .WillOnce(Return(canceled)); + } + + void expect_notify_sync_complete(MockInstanceWatcher &mock_instance_watcher, + const std::string &sync_id) { + EXPECT_CALL(mock_instance_watcher, notify_sync_complete(sync_id)); + } + + void expect_create_sync_point(librbd::MockTestImageCtx &mock_local_image_ctx, + MockSyncPointCreateRequest &mock_sync_point_create_request, + int r) { + EXPECT_CALL(mock_sync_point_create_request, send()) + .WillOnce(Invoke([this, &mock_local_image_ctx, &mock_sync_point_create_request, r]() { + if (r == 0) { + mock_local_image_ctx.snap_ids[{cls::rbd::UserSnapshotNamespace(), + "snap1"}] = 123; + m_client_meta.sync_points.emplace_back(cls::rbd::UserSnapshotNamespace(), + "snap1", + boost::none); + } + m_threads->work_queue->queue(mock_sync_point_create_request.on_finish, r); + })); + } + + void expect_copy_image(MockImageCopyRequest &mock_image_copy_request, int r) { + EXPECT_CALL(mock_image_copy_request, send()) + .WillOnce(Invoke([this, &mock_image_copy_request, r]() { + m_threads->work_queue->queue(mock_image_copy_request.on_finish, r); + })); + } + + void expect_flush_sync_point(journal::MockJournaler &mock_journaler, int r) { + EXPECT_CALL(mock_journaler, update_client(_, _)) + .WillOnce(WithArg<1>(CompleteContext(r))); + } + + void expect_prune_sync_point(MockSyncPointPruneRequest &mock_sync_point_prune_request, + bool sync_complete, int r) { + EXPECT_CALL(mock_sync_point_prune_request, send()) + .WillOnce(Invoke([this, &mock_sync_point_prune_request, sync_complete, r]() { + ASSERT_EQ(sync_complete, mock_sync_point_prune_request.sync_complete); + if (r == 0 && !m_client_meta.sync_points.empty()) { + if (sync_complete) { + m_client_meta.sync_points.pop_front(); + } else { + while (m_client_meta.sync_points.size() > 1) { + m_client_meta.sync_points.pop_back(); + } + } + } + m_threads->work_queue->queue(mock_sync_point_prune_request.on_finish, r); + })); + } + + MockImageSync *create_request(librbd::MockTestImageCtx &mock_remote_image_ctx, + librbd::MockTestImageCtx &mock_local_image_ctx, + journal::MockJournaler &mock_journaler, + MockInstanceWatcher &mock_instance_watcher, + Context *ctx) { + return new MockImageSync(&mock_local_image_ctx, &mock_remote_image_ctx, + m_threads->timer, &m_threads->timer_lock, + "mirror-uuid", &mock_journaler, &m_client_meta, + m_threads->work_queue, &mock_instance_watcher, + ctx); + } + + librbd::ImageCtx *m_remote_image_ctx; + librbd::ImageCtx *m_local_image_ctx; + librbd::journal::MirrorPeerClientMeta m_client_meta; +}; + +TEST_F(TestMockImageSync, SimpleSync) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + journal::MockJournaler mock_journaler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_create_sync_point(mock_local_image_ctx, mock_sync_point_create_request, 0); + expect_get_snap_id(mock_remote_image_ctx); + expect_copy_image(mock_image_copy_request, 0); + expect_flush_sync_point(mock_journaler, 0); + expect_prune_sync_point(mock_sync_point_prune_request, true, 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_remote_image_ctx, + mock_local_image_ctx, mock_journaler, + mock_instance_watcher, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageSync, RestartSync) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + journal::MockJournaler mock_journaler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + m_client_meta.sync_points = {{cls::rbd::UserSnapshotNamespace(), "snap1", boost::none}, + {cls::rbd::UserSnapshotNamespace(), "snap2", "snap1", boost::none}}; + mock_local_image_ctx.snap_ids[{cls::rbd::UserSnapshotNamespace(), "snap1"}] = 123; + mock_local_image_ctx.snap_ids[{cls::rbd::UserSnapshotNamespace(), "snap2"}] = 234; + + expect_test_features(mock_local_image_ctx); + + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_prune_sync_point(mock_sync_point_prune_request, false, 0); + expect_get_snap_id(mock_remote_image_ctx); + expect_copy_image(mock_image_copy_request, 0); + expect_flush_sync_point(mock_journaler, 0); + expect_prune_sync_point(mock_sync_point_prune_request, true, 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_remote_image_ctx, + mock_local_image_ctx, mock_journaler, + mock_instance_watcher, &ctx); + request->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageSync, CancelNotifySyncRequest) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + journal::MockJournaler mock_journaler; + MockInstanceWatcher mock_instance_watcher; + + InSequence seq; + Context *on_sync_start = nullptr; + C_SaferCond notify_sync_ctx; + EXPECT_CALL(mock_instance_watcher, + notify_sync_request(mock_local_image_ctx.id, _)) + .WillOnce(Invoke([&on_sync_start, ¬ify_sync_ctx]( + const std::string &, Context *ctx) { + on_sync_start = ctx; + notify_sync_ctx.complete(0); + })); + EXPECT_CALL(mock_instance_watcher, + cancel_sync_request(mock_local_image_ctx.id)) + .WillOnce(Invoke([&on_sync_start](const std::string &) { + EXPECT_NE(nullptr, on_sync_start); + on_sync_start->complete(-ECANCELED); + return true; + })); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_remote_image_ctx, + mock_local_image_ctx, mock_journaler, + mock_instance_watcher, &ctx); + request->get(); + request->send(); + + // cancel the notify sync request once it starts + ASSERT_EQ(0, notify_sync_ctx.wait()); + request->cancel(); + request->put(); + + ASSERT_EQ(-ECANCELED, ctx.wait()); +} + +TEST_F(TestMockImageSync, CancelImageCopy) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + journal::MockJournaler mock_journaler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + m_client_meta.sync_points = {{cls::rbd::UserSnapshotNamespace(), "snap1", boost::none}}; + + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_prune_sync_point(mock_sync_point_prune_request, false, 0); + expect_get_snap_id(mock_remote_image_ctx); + + C_SaferCond image_copy_ctx; + EXPECT_CALL(mock_image_copy_request, send()) + .WillOnce(Invoke([&image_copy_ctx]() { + image_copy_ctx.complete(0); + })); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id, + false); + EXPECT_CALL(mock_image_copy_request, cancel()); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_remote_image_ctx, + mock_local_image_ctx, mock_journaler, + mock_instance_watcher, &ctx); + request->get(); + request->send(); + + // cancel the image copy once it starts + ASSERT_EQ(0, image_copy_ctx.wait()); + request->cancel(); + request->put(); + m_threads->work_queue->queue(mock_image_copy_request.on_finish, 0); + + ASSERT_EQ(-ECANCELED, ctx.wait()); +} + +TEST_F(TestMockImageSync, CancelAfterCopyImage) { + librbd::MockTestImageCtx mock_remote_image_ctx(*m_remote_image_ctx); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + journal::MockJournaler mock_journaler; + MockInstanceWatcher mock_instance_watcher; + MockImageCopyRequest mock_image_copy_request; + MockSyncPointCreateRequest mock_sync_point_create_request; + MockSyncPointPruneRequest mock_sync_point_prune_request; + + C_SaferCond ctx; + MockImageSync *request = create_request(mock_remote_image_ctx, + mock_local_image_ctx, mock_journaler, + mock_instance_watcher, &ctx); + InSequence seq; + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); + expect_create_sync_point(mock_local_image_ctx, mock_sync_point_create_request, 0); + expect_get_snap_id(mock_remote_image_ctx); + EXPECT_CALL(mock_image_copy_request, send()) + .WillOnce((DoAll(InvokeWithoutArgs([request]() { + request->cancel(); + }), + Invoke([this, &mock_image_copy_request]() { + m_threads->work_queue->queue(mock_image_copy_request.on_finish, 0); + })))); + expect_cancel_sync_request(mock_instance_watcher, mock_local_image_ctx.id, + false); + EXPECT_CALL(mock_image_copy_request, cancel()); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); + + request->send(); + ASSERT_EQ(-ECANCELED, ctx.wait()); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_ImageSyncThrottler.cc b/src/test/rbd_mirror/test_mock_ImageSyncThrottler.cc new file mode 100644 index 00000000..af88edcb --- /dev/null +++ b/src/test/rbd_mirror/test_mock_ImageSyncThrottler.cc @@ -0,0 +1,241 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2016 SUSE LINUX GmbH + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +// template definitions +#include "tools/rbd_mirror/ImageSyncThrottler.cc" + +namespace rbd { +namespace mirror { + +class TestMockImageSyncThrottler : public TestMockFixture { +public: + typedef ImageSyncThrottler<librbd::MockTestImageCtx> MockImageSyncThrottler; + +}; + +TEST_F(TestMockImageSyncThrottler, Single_Sync) { + MockImageSyncThrottler throttler(g_ceph_context); + C_SaferCond on_start; + throttler.start_op("id", &on_start); + ASSERT_EQ(0, on_start.wait()); + throttler.finish_op("id"); +} + +TEST_F(TestMockImageSyncThrottler, Multiple_Syncs) { + MockImageSyncThrottler throttler(g_ceph_context); + throttler.set_max_concurrent_syncs(2); + + C_SaferCond on_start1; + throttler.start_op("id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("id2", &on_start2); + C_SaferCond on_start3; + throttler.start_op("id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("id4", &on_start4); + + ASSERT_EQ(0, on_start2.wait()); + throttler.finish_op("id2"); + ASSERT_EQ(0, on_start3.wait()); + throttler.finish_op("id3"); + ASSERT_EQ(0, on_start1.wait()); + throttler.finish_op("id1"); + ASSERT_EQ(0, on_start4.wait()); + throttler.finish_op("id4"); +} + +TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync) { + MockImageSyncThrottler throttler(g_ceph_context); + C_SaferCond on_start; + throttler.start_op("id", &on_start); + ASSERT_EQ(0, on_start.wait()); + ASSERT_FALSE(throttler.cancel_op("id")); + throttler.finish_op("id"); +} + +TEST_F(TestMockImageSyncThrottler, Cancel_Waiting_Sync) { + MockImageSyncThrottler throttler(g_ceph_context); + throttler.set_max_concurrent_syncs(1); + + C_SaferCond on_start1; + throttler.start_op("id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("id2", &on_start2); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_TRUE(throttler.cancel_op("id2")); + ASSERT_EQ(-ECANCELED, on_start2.wait()); + throttler.finish_op("id1"); +} + + +TEST_F(TestMockImageSyncThrottler, Cancel_Running_Sync_Start_Waiting) { + MockImageSyncThrottler throttler(g_ceph_context); + throttler.set_max_concurrent_syncs(1); + + C_SaferCond on_start1; + throttler.start_op("id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("id2", &on_start2); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_FALSE(throttler.cancel_op("id1")); + throttler.finish_op("id1"); + ASSERT_EQ(0, on_start2.wait()); + throttler.finish_op("id2"); +} + +TEST_F(TestMockImageSyncThrottler, Duplicate) { + MockImageSyncThrottler throttler(g_ceph_context); + throttler.set_max_concurrent_syncs(1); + + C_SaferCond on_start1; + throttler.start_op("id1", &on_start1); + ASSERT_EQ(0, on_start1.wait()); + + C_SaferCond on_start2; + throttler.start_op("id1", &on_start2); + ASSERT_EQ(0, on_start2.wait()); + + C_SaferCond on_start3; + throttler.start_op("id2", &on_start3); + C_SaferCond on_start4; + throttler.start_op("id2", &on_start4); + ASSERT_EQ(-ENOENT, on_start3.wait()); + + throttler.finish_op("id1"); + ASSERT_EQ(0, on_start4.wait()); + throttler.finish_op("id2"); +} + +TEST_F(TestMockImageSyncThrottler, Duplicate2) { + MockImageSyncThrottler throttler(g_ceph_context); + throttler.set_max_concurrent_syncs(2); + + C_SaferCond on_start1; + throttler.start_op("id1", &on_start1); + ASSERT_EQ(0, on_start1.wait()); + C_SaferCond on_start2; + throttler.start_op("id2", &on_start2); + ASSERT_EQ(0, on_start2.wait()); + + C_SaferCond on_start3; + throttler.start_op("id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("id3", &on_start4); // dup + ASSERT_EQ(-ENOENT, on_start3.wait()); + + C_SaferCond on_start5; + throttler.start_op("id4", &on_start5); + + throttler.finish_op("id1"); + ASSERT_EQ(0, on_start4.wait()); + + throttler.finish_op("id2"); + ASSERT_EQ(0, on_start5.wait()); + + C_SaferCond on_start6; + throttler.start_op("id5", &on_start6); + + throttler.finish_op("id3"); + ASSERT_EQ(0, on_start6.wait()); + + throttler.finish_op("id4"); + throttler.finish_op("id5"); +} + +TEST_F(TestMockImageSyncThrottler, Increase_Max_Concurrent_Syncs) { + MockImageSyncThrottler throttler(g_ceph_context); + throttler.set_max_concurrent_syncs(2); + + C_SaferCond on_start1; + throttler.start_op("id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("id2", &on_start2); + C_SaferCond on_start3; + throttler.start_op("id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("id4", &on_start4); + C_SaferCond on_start5; + throttler.start_op("id5", &on_start5); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_EQ(0, on_start2.wait()); + + throttler.set_max_concurrent_syncs(4); + + ASSERT_EQ(0, on_start3.wait()); + ASSERT_EQ(0, on_start4.wait()); + + throttler.finish_op("id4"); + ASSERT_EQ(0, on_start5.wait()); + + throttler.finish_op("id1"); + throttler.finish_op("id2"); + throttler.finish_op("id3"); + throttler.finish_op("id5"); +} + +TEST_F(TestMockImageSyncThrottler, Decrease_Max_Concurrent_Syncs) { + MockImageSyncThrottler throttler(g_ceph_context); + throttler.set_max_concurrent_syncs(4); + + C_SaferCond on_start1; + throttler.start_op("id1", &on_start1); + C_SaferCond on_start2; + throttler.start_op("id2", &on_start2); + C_SaferCond on_start3; + throttler.start_op("id3", &on_start3); + C_SaferCond on_start4; + throttler.start_op("id4", &on_start4); + C_SaferCond on_start5; + throttler.start_op("id5", &on_start5); + + ASSERT_EQ(0, on_start1.wait()); + ASSERT_EQ(0, on_start2.wait()); + ASSERT_EQ(0, on_start3.wait()); + ASSERT_EQ(0, on_start4.wait()); + + throttler.set_max_concurrent_syncs(2); + + throttler.finish_op("id1"); + throttler.finish_op("id2"); + throttler.finish_op("id3"); + + ASSERT_EQ(0, on_start5.wait()); + + throttler.finish_op("id4"); + throttler.finish_op("id5"); +} + +} // namespace mirror +} // namespace rbd + diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc new file mode 100644 index 00000000..79dce7ac --- /dev/null +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -0,0 +1,366 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "tools/rbd_mirror/ImageReplayer.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/InstanceReplayer.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/Types.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + Mutex &timer_lock; + Cond timer_cond; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +template<> +struct ServiceDaemon<librbd::MockTestImageCtx> { + MOCK_METHOD3(add_or_update_attribute, + void(int64_t, const std::string&, + const service_daemon::AttributeValue&)); +}; + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { +}; + +template<> +struct ImageReplayer<librbd::MockTestImageCtx> { + static ImageReplayer* s_instance; + std::string global_image_id; + + static ImageReplayer *create( + Threads<librbd::MockTestImageCtx> *threads, + InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher, + RadosRef local, const std::string &local_mirror_uuid, int64_t local_pool_id, + const std::string &global_image_id) { + ceph_assert(s_instance != nullptr); + s_instance->global_image_id = global_image_id; + return s_instance; + } + + ImageReplayer() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + virtual ~ImageReplayer() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD2(start, void(Context *, bool)); + MOCK_METHOD2(stop, void(Context *, bool)); + MOCK_METHOD0(restart, void()); + MOCK_METHOD0(flush, void()); + MOCK_METHOD2(print_status, void(Formatter *, stringstream *)); + MOCK_METHOD2(add_peer, void(const std::string &, librados::IoCtx &)); + MOCK_METHOD0(get_global_image_id, const std::string &()); + MOCK_METHOD0(get_local_image_id, const std::string &()); + MOCK_METHOD0(is_running, bool()); + MOCK_METHOD0(is_stopped, bool()); + MOCK_METHOD0(is_blacklisted, bool()); + + MOCK_CONST_METHOD0(is_finished, bool()); + MOCK_METHOD1(set_finished, void(bool)); + + MOCK_CONST_METHOD0(get_health_state, image_replayer::HealthState()); +}; + +ImageReplayer<librbd::MockTestImageCtx>* ImageReplayer<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/InstanceReplayer.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::ReturnRef; +using ::testing::WithArg; + +class TestMockInstanceReplayer : public TestMockFixture { +public: + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef ImageReplayer<librbd::MockTestImageCtx> MockImageReplayer; + typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon; + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillOnce(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_add_event_after(MockThreads &mock_threads, + Context** timer_ctx = nullptr) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce(DoAll( + WithArg<1>(Invoke([this, &mock_threads, timer_ctx](Context *ctx) { + ceph_assert(mock_threads.timer_lock.is_locked()); + if (timer_ctx != nullptr) { + *timer_ctx = ctx; + mock_threads.timer_cond.SignalOne(); + } else { + m_threads->work_queue->queue( + new FunctionContext([&mock_threads, ctx](int) { + Mutex::Locker timer_lock(mock_threads.timer_lock); + ctx->complete(0); + }), 0); + } + })), + ReturnArg<1>())); + } + + void expect_cancel_event(MockThreads &mock_threads, bool canceled) { + EXPECT_CALL(*mock_threads.timer, cancel_event(_)) + .WillOnce(Return(canceled)); + } +}; + +TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) { + MockThreads mock_threads(m_threads); + MockServiceDaemon mock_service_daemon; + MockInstanceWatcher mock_instance_watcher; + MockImageReplayer mock_image_replayer; + MockInstanceReplayer instance_replayer( + &mock_threads, &mock_service_daemon, + rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)), + "local_mirror_uuid", m_local_io_ctx.get_id()); + std::string global_image_id("global_image_id"); + + EXPECT_CALL(mock_image_replayer, get_global_image_id()) + .WillRepeatedly(ReturnRef(global_image_id)); + + InSequence seq; + expect_work_queue(mock_threads); + Context *timer_ctx = nullptr; + expect_add_event_after(mock_threads, &timer_ctx); + instance_replayer.init(); + instance_replayer.add_peer("peer_uuid", m_remote_io_ctx); + + // Acquire + + C_SaferCond on_acquire; + EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, start(nullptr, false)); + expect_work_queue(mock_threads); + + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + // Release + + C_SaferCond on_release; + + EXPECT_CALL(mock_image_replayer, is_stopped()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_running()) + .WillOnce(Return(false)); + expect_work_queue(mock_threads); + expect_add_event_after(mock_threads); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, is_stopped()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_running()) + .WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, stop(_, false)) + .WillOnce(CompleteContext(0)); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, is_stopped()) + .WillOnce(Return(true)); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, destroy()); + + instance_replayer.release_image("global_image_id", &on_release); + ASSERT_EQ(0, on_release.wait()); + + expect_work_queue(mock_threads); + expect_cancel_event(mock_threads, true); + expect_work_queue(mock_threads); + instance_replayer.shut_down(); + ASSERT_TRUE(timer_ctx != nullptr); + delete timer_ctx; +} + +TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) { + MockThreads mock_threads(m_threads); + MockServiceDaemon mock_service_daemon; + MockInstanceWatcher mock_instance_watcher; + MockImageReplayer mock_image_replayer; + MockInstanceReplayer instance_replayer( + &mock_threads, &mock_service_daemon, + rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)), + "local_mirror_uuid", m_local_io_ctx.get_id()); + std::string global_image_id("global_image_id"); + + EXPECT_CALL(mock_image_replayer, get_global_image_id()) + .WillRepeatedly(ReturnRef(global_image_id)); + + InSequence seq; + expect_work_queue(mock_threads); + Context *timer_ctx1 = nullptr; + expect_add_event_after(mock_threads, &timer_ctx1); + instance_replayer.init(); + instance_replayer.add_peer("peer_uuid", m_remote_io_ctx); + + // Acquire + + C_SaferCond on_acquire; + EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, start(nullptr, false)); + expect_work_queue(mock_threads); + + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire); + ASSERT_EQ(0, on_acquire.wait()); + + // periodic start timer + Context *timer_ctx2 = nullptr; + expect_add_event_after(mock_threads, &timer_ctx2); + + Context *start_image_replayers_ctx = nullptr; + EXPECT_CALL(*mock_threads.work_queue, queue(_, 0)) + .WillOnce(Invoke([&start_image_replayers_ctx](Context *ctx, int r) { + start_image_replayers_ctx = ctx; + })); + + ASSERT_TRUE(timer_ctx1 != nullptr); + { + Mutex::Locker timer_locker(mock_threads.timer_lock); + timer_ctx1->complete(0); + } + + // remove finished image replayer + EXPECT_CALL(mock_image_replayer, get_health_state()).WillOnce( + Return(image_replayer::HEALTH_STATE_OK)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, destroy()); + EXPECT_CALL(mock_service_daemon,add_or_update_attribute(_, _, _)).Times(3); + + ASSERT_TRUE(start_image_replayers_ctx != nullptr); + start_image_replayers_ctx->complete(0); + + // shut down + expect_work_queue(mock_threads); + expect_cancel_event(mock_threads, true); + expect_work_queue(mock_threads); + instance_replayer.shut_down(); + ASSERT_TRUE(timer_ctx2 != nullptr); + delete timer_ctx2; +} + +TEST_F(TestMockInstanceReplayer, Reacquire) { + MockThreads mock_threads(m_threads); + MockServiceDaemon mock_service_daemon; + MockInstanceWatcher mock_instance_watcher; + MockImageReplayer mock_image_replayer; + MockInstanceReplayer instance_replayer( + &mock_threads, &mock_service_daemon, + rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)), + "local_mirror_uuid", m_local_io_ctx.get_id()); + std::string global_image_id("global_image_id"); + + EXPECT_CALL(mock_image_replayer, get_global_image_id()) + .WillRepeatedly(ReturnRef(global_image_id)); + + InSequence seq; + expect_work_queue(mock_threads); + Context *timer_ctx = nullptr; + expect_add_event_after(mock_threads, &timer_ctx); + instance_replayer.init(); + instance_replayer.add_peer("peer_uuid", m_remote_io_ctx); + + // Acquire + + EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _)); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false)); + EXPECT_CALL(mock_image_replayer, start(nullptr, false)); + expect_work_queue(mock_threads); + + C_SaferCond on_acquire1; + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire1); + ASSERT_EQ(0, on_acquire1.wait()); + + // Re-acquire + EXPECT_CALL(mock_image_replayer, set_finished(false)); + EXPECT_CALL(mock_image_replayer, restart()); + expect_work_queue(mock_threads); + + C_SaferCond on_acquire2; + instance_replayer.acquire_image(&mock_instance_watcher, global_image_id, + &on_acquire2); + ASSERT_EQ(0, on_acquire2.wait()); + + expect_work_queue(mock_threads); + expect_cancel_event(mock_threads, true); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + expect_work_queue(mock_threads); + expect_work_queue(mock_threads); + EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true)); + EXPECT_CALL(mock_image_replayer, destroy()); + instance_replayer.shut_down(); + ASSERT_TRUE(timer_ctx != nullptr); + delete timer_ctx; +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc new file mode 100644 index 00000000..075134d9 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc @@ -0,0 +1,988 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librados/AioCompletionImpl.h" +#include "librbd/ManagedLock.h" +#include "test/librados/test_cxx.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "tools/rbd_mirror/InstanceReplayer.h" +#include "tools/rbd_mirror/ImageSyncThrottler.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/Threads.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +template <> +struct ManagedLock<MockTestImageCtx> { + static ManagedLock* s_instance; + + static ManagedLock *create(librados::IoCtx& ioctx, ContextWQ *work_queue, + const std::string& oid, librbd::Watcher *watcher, + managed_lock::Mode mode, + bool blacklist_on_break_lock, + uint32_t blacklist_expire_seconds) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + ManagedLock() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~ManagedLock() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD1(shut_down, void(Context *)); + MOCK_METHOD1(acquire_lock, void(Context *)); + MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *)); + MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *)); +}; + +ManagedLock<MockTestImageCtx> *ManagedLock<MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +template <> +struct InstanceReplayer<librbd::MockTestImageCtx> { + MOCK_METHOD3(acquire_image, void(InstanceWatcher<librbd::MockTestImageCtx> *, + const std::string &, Context *)); + MOCK_METHOD2(release_image, void(const std::string &, Context *)); + MOCK_METHOD3(remove_peer_image, void(const std::string&, const std::string&, + Context *)); +}; + +template <> +struct ImageSyncThrottler<librbd::MockTestImageCtx> { + static ImageSyncThrottler* s_instance; + + static ImageSyncThrottler *create(CephContext *cct) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + ImageSyncThrottler() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + virtual ~ImageSyncThrottler() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD1(drain, void(int)); + MOCK_METHOD2(start_op, void(const std::string &, Context *)); + MOCK_METHOD1(finish_op, void(const std::string &)); +}; + +ImageSyncThrottler<librbd::MockTestImageCtx>* ImageSyncThrottler<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/InstanceWatcher.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockInstanceWatcher : public TestMockFixture { +public: + typedef librbd::ManagedLock<librbd::MockTestImageCtx> MockManagedLock; + typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + std::string m_instance_id; + std::string m_oid; + MockThreads *m_mock_threads; + + void SetUp() override { + TestFixture::SetUp(); + m_local_io_ctx.remove(RBD_MIRROR_LEADER); + EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true)); + + m_instance_id = stringify(m_local_io_ctx.get_instance_id()); + m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id; + + m_mock_threads = new MockThreads(m_threads); + } + + void TearDown() override { + delete m_mock_threads; + TestMockFixture::TearDown(); + } + + void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { + EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _)); + } + + void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx, + const std::string &instance_id) { + std::string oid = RBD_MIRROR_INSTANCE_PREFIX + instance_id; + EXPECT_CALL(mock_io_ctx, aio_watch(oid, _, _, _)); + } + + void expect_unregister_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) { + EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _)); + } + + void expect_register_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, + int r) { + EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), + StrEq("mirror_instances_add"), _, _, _)) + .WillOnce(Return(r)); + } + + void expect_unregister_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx, + int r) { + EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"), + StrEq("mirror_instances_remove"), _, _, _)) + .WillOnce(Return(r)); + } + + void expect_acquire_lock(MockManagedLock &mock_managed_lock, int r) { + EXPECT_CALL(mock_managed_lock, acquire_lock(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_release_lock(MockManagedLock &mock_managed_lock, int r) { + EXPECT_CALL(mock_managed_lock, shut_down(_)).WillOnce(CompleteContext(r)); + } + + void expect_destroy_lock(MockManagedLock &mock_managed_lock, + Context *ctx = nullptr) { + EXPECT_CALL(mock_managed_lock, destroy()) + .WillOnce(Invoke([ctx]() { + if (ctx != nullptr) { + ctx->complete(0); + } + })); + } + + void expect_get_locker(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r) { + EXPECT_CALL(mock_managed_lock, get_locker(_, _)) + .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out, + Context *ctx) { + if (r == 0) { + *out = locker; + } + ctx->complete(r); + })); + } + + void expect_break_lock(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r) { + EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _)) + .WillOnce(WithArg<2>(CompleteContext(r))); + } +}; + +TEST_F(TestMockInstanceWatcher, InitShutdown) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, InitError) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); + InSequence seq; + + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, -EINVAL); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + + ASSERT_EQ(-EINVAL, instance_watcher->init()); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, ShutdownError) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Shutdown + expect_release_lock(mock_managed_lock, -EINVAL); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + + +TEST_F(TestMockInstanceWatcher, Remove) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + librbd::managed_lock::Locker + locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123}; + + InSequence seq; + + expect_get_locker(mock_managed_lock, locker, 0); + expect_break_lock(mock_managed_lock, locker, 0); + expect_unregister_instance(mock_io_ctx, 0); + C_SaferCond on_destroy; + expect_destroy_lock(mock_managed_lock, &on_destroy); + + C_SaferCond on_remove; + MockInstanceWatcher::remove_instance(m_local_io_ctx, + m_mock_threads->work_queue, + "instance_id", &on_remove); + ASSERT_EQ(0, on_remove.wait()); + ASSERT_EQ(0, on_destroy.wait()); +} + +TEST_F(TestMockInstanceWatcher, RemoveNoent) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + InSequence seq; + + expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT); + expect_unregister_instance(mock_io_ctx, 0); + C_SaferCond on_destroy; + expect_destroy_lock(mock_managed_lock, &on_destroy); + + C_SaferCond on_remove; + MockInstanceWatcher::remove_instance(m_local_io_ctx, + m_mock_threads->work_queue, + "instance_id", &on_remove); + ASSERT_EQ(0, on_remove.wait()); + ASSERT_EQ(0, on_destroy.wait()); +} + +TEST_F(TestMockInstanceWatcher, ImageAcquireRelease) { + MockManagedLock mock_managed_lock; + + librados::IoCtx& io_ctx1 = m_local_io_ctx; + std::string instance_id1 = m_instance_id; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + MockInstanceReplayer mock_instance_replayer1; + auto instance_watcher1 = MockInstanceWatcher::create( + io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1); + + librados::Rados cluster; + librados::IoCtx io_ctx2; + EXPECT_EQ("", connect_cluster_pp(cluster)); + EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); + std::string instance_id2 = stringify(io_ctx2.get_instance_id()); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + MockInstanceReplayer mock_instance_replayer2; + auto instance_watcher2 = MockInstanceWatcher::create( + io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2); + + InSequence seq; + + // Init instance watcher 1 + expect_register_instance(mock_io_ctx1, 0); + expect_register_watch(mock_io_ctx1, instance_id1); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher1->init()); + + // Init instance watcher 2 + expect_register_instance(mock_io_ctx2, 0); + expect_register_watch(mock_io_ctx2, instance_id2); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher2->init()); + + // Acquire Image on the the same instance + EXPECT_CALL(mock_instance_replayer1, acquire_image(instance_watcher1, "gid", + _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_acquire1; + instance_watcher1->notify_image_acquire(instance_id1, "gid", &on_acquire1); + ASSERT_EQ(0, on_acquire1.wait()); + + // Acquire Image on the other instance + EXPECT_CALL(mock_instance_replayer2, acquire_image(instance_watcher2, "gid", + _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_acquire2; + instance_watcher1->notify_image_acquire(instance_id2, "gid", &on_acquire2); + ASSERT_EQ(0, on_acquire2.wait()); + + // Release Image on the the same instance + EXPECT_CALL(mock_instance_replayer1, release_image("gid", _)) + .WillOnce(WithArg<1>(CompleteContext(0))); + C_SaferCond on_release1; + instance_watcher1->notify_image_release(instance_id1, "gid", &on_release1); + ASSERT_EQ(0, on_release1.wait()); + + // Release Image on the other instance + EXPECT_CALL(mock_instance_replayer2, release_image("gid", _)) + .WillOnce(WithArg<1>(CompleteContext(0))); + C_SaferCond on_release2; + instance_watcher1->notify_image_release(instance_id2, "gid", &on_release2); + ASSERT_EQ(0, on_release2.wait()); + + // Shutdown instance watcher 1 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx1); + expect_unregister_instance(mock_io_ctx1, 0); + instance_watcher1->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher1; + + // Shutdown instance watcher 2 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx2); + expect_unregister_instance(mock_io_ctx2, 0); + instance_watcher2->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher2; +} + +TEST_F(TestMockInstanceWatcher, PeerImageRemoved) { + MockManagedLock mock_managed_lock; + + librados::IoCtx& io_ctx1 = m_local_io_ctx; + std::string instance_id1 = m_instance_id; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + MockInstanceReplayer mock_instance_replayer1; + auto instance_watcher1 = MockInstanceWatcher::create( + io_ctx1, m_mock_threads->work_queue, &mock_instance_replayer1); + + librados::Rados cluster; + librados::IoCtx io_ctx2; + EXPECT_EQ("", connect_cluster_pp(cluster)); + EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); + std::string instance_id2 = stringify(io_ctx2.get_instance_id()); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + MockInstanceReplayer mock_instance_replayer2; + auto instance_watcher2 = MockInstanceWatcher::create( + io_ctx2, m_mock_threads->work_queue, &mock_instance_replayer2); + + InSequence seq; + + // Init instance watcher 1 + expect_register_instance(mock_io_ctx1, 0); + expect_register_watch(mock_io_ctx1, instance_id1); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher1->init()); + + // Init instance watcher 2 + expect_register_instance(mock_io_ctx2, 0); + expect_register_watch(mock_io_ctx2, instance_id2); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher2->init()); + + // Peer Image Removed on the same instance + EXPECT_CALL(mock_instance_replayer1, remove_peer_image("gid", "uuid", _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_removed1; + instance_watcher1->notify_peer_image_removed(instance_id1, "gid", "uuid", + &on_removed1); + ASSERT_EQ(0, on_removed1.wait()); + + // Peer Image Removed on the other instance + EXPECT_CALL(mock_instance_replayer2, remove_peer_image("gid", "uuid", _)) + .WillOnce(WithArg<2>(CompleteContext(0))); + C_SaferCond on_removed2; + instance_watcher1->notify_peer_image_removed(instance_id2, "gid", "uuid", + &on_removed2); + ASSERT_EQ(0, on_removed2.wait()); + + // Shutdown instance watcher 1 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx1); + expect_unregister_instance(mock_io_ctx1, 0); + instance_watcher1->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher1; + + // Shutdown instance watcher 2 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx2); + expect_unregister_instance(mock_io_ctx2, 0); + instance_watcher2->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher2; +} + +TEST_F(TestMockInstanceWatcher, ImageAcquireReleaseCancel) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Send Acquire Image and cancel + EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) + .WillOnce(Invoke( + [this, instance_watcher, &mock_io_ctx]( + const std::string& o, librados::AioCompletionImpl *c, + bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { + c->get(); + auto ctx = new FunctionContext( + [instance_watcher, &mock_io_ctx, c, pbl](int r) { + instance_watcher->cancel_notify_requests("other"); + encode(librbd::watcher::NotifyResponse(), *pbl); + mock_io_ctx.get_mock_rados_client()-> + finish_aio_completion(c, -ETIMEDOUT); + }); + m_threads->work_queue->queue(ctx, 0); + })); + + C_SaferCond on_acquire; + instance_watcher->notify_image_acquire("other", "gid", &on_acquire); + ASSERT_EQ(-ECANCELED, on_acquire.wait()); + + // Send Release Image and cancel + EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) + .WillOnce(Invoke( + [this, instance_watcher, &mock_io_ctx]( + const std::string& o, librados::AioCompletionImpl *c, + bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { + c->get(); + auto ctx = new FunctionContext( + [instance_watcher, &mock_io_ctx, c, pbl](int r) { + instance_watcher->cancel_notify_requests("other"); + encode(librbd::watcher::NotifyResponse(), *pbl); + mock_io_ctx.get_mock_rados_client()-> + finish_aio_completion(c, -ETIMEDOUT); + }); + m_threads->work_queue->queue(ctx, 0); + })); + + C_SaferCond on_release; + instance_watcher->notify_image_release("other", "gid", &on_release); + ASSERT_EQ(-ECANCELED, on_release.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, PeerImageAcquireWatchDNE) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + MockInstanceReplayer mock_instance_replayer; + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer, + m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Acquire image on dead (blacklisted) instance + C_SaferCond on_acquire; + instance_watcher->notify_image_acquire("dead instance", "global image id", + &on_acquire); + ASSERT_EQ(-ENOENT, on_acquire.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, PeerImageReleaseWatchDNE) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + MockInstanceReplayer mock_instance_replayer; + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, m_mock_threads->work_queue, &mock_instance_replayer, + m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Release image on dead (blacklisted) instance + C_SaferCond on_acquire; + instance_watcher->notify_image_release("dead instance", "global image id", + &on_acquire); + ASSERT_EQ(-ENOENT, on_acquire.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +TEST_F(TestMockInstanceWatcher, PeerImageRemovedCancel) { + MockManagedLock mock_managed_lock; + librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx)); + + auto instance_watcher = new MockInstanceWatcher( + m_local_io_ctx, m_mock_threads->work_queue, nullptr, m_instance_id); + InSequence seq; + + // Init + expect_register_instance(mock_io_ctx, 0); + expect_register_watch(mock_io_ctx); + expect_acquire_lock(mock_managed_lock, 0); + ASSERT_EQ(0, instance_watcher->init()); + + // Send Acquire Image and cancel + EXPECT_CALL(mock_io_ctx, aio_notify(_, _, _, _, _)) + .WillOnce(Invoke( + [this, instance_watcher, &mock_io_ctx]( + const std::string& o, librados::AioCompletionImpl *c, + bufferlist& bl, uint64_t timeout_ms, bufferlist *pbl) { + c->get(); + auto ctx = new FunctionContext( + [instance_watcher, &mock_io_ctx, c, pbl](int r) { + instance_watcher->cancel_notify_requests("other"); + encode(librbd::watcher::NotifyResponse(), *pbl); + mock_io_ctx.get_mock_rados_client()-> + finish_aio_completion(c, -ETIMEDOUT); + }); + m_threads->work_queue->queue(ctx, 0); + })); + + C_SaferCond on_acquire; + instance_watcher->notify_peer_image_removed("other", "gid", "uuid", + &on_acquire); + ASSERT_EQ(-ECANCELED, on_acquire.wait()); + + // Shutdown + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx); + expect_unregister_instance(mock_io_ctx, 0); + instance_watcher->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher; +} + +class TestMockInstanceWatcher_NotifySync : public TestMockInstanceWatcher { +public: + typedef ImageSyncThrottler<librbd::MockTestImageCtx> MockImageSyncThrottler; + + MockManagedLock mock_managed_lock; + MockImageSyncThrottler mock_image_sync_throttler; + std::string instance_id1; + std::string instance_id2; + + librados::Rados cluster; + librados::IoCtx io_ctx2; + + MockInstanceWatcher *instance_watcher1; + MockInstanceWatcher *instance_watcher2; + + void SetUp() override { + TestMockInstanceWatcher::SetUp(); + + instance_id1 = m_instance_id; + librados::IoCtx& io_ctx1 = m_local_io_ctx; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + instance_watcher1 = MockInstanceWatcher::create(io_ctx1, + m_mock_threads->work_queue, + nullptr); + EXPECT_EQ("", connect_cluster_pp(cluster)); + EXPECT_EQ(0, cluster.ioctx_create(_local_pool_name.c_str(), io_ctx2)); + instance_id2 = stringify(io_ctx2.get_instance_id()); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + instance_watcher2 = MockInstanceWatcher::create(io_ctx2, + m_mock_threads->work_queue, + nullptr); + InSequence seq; + + // Init instance watcher 1 (leader) + expect_register_instance(mock_io_ctx1, 0); + expect_register_watch(mock_io_ctx1, instance_id1); + expect_acquire_lock(mock_managed_lock, 0); + EXPECT_EQ(0, instance_watcher1->init()); + instance_watcher1->handle_acquire_leader(); + + // Init instance watcher 2 + expect_register_instance(mock_io_ctx2, 0); + expect_register_watch(mock_io_ctx2, instance_id2); + expect_acquire_lock(mock_managed_lock, 0); + EXPECT_EQ(0, instance_watcher2->init()); + instance_watcher2->handle_update_leader(instance_id1); + } + + void TearDown() override { + librados::IoCtx& io_ctx1 = m_local_io_ctx; + librados::MockTestMemIoCtxImpl &mock_io_ctx1(get_mock_io_ctx(io_ctx1)); + librados::MockTestMemIoCtxImpl &mock_io_ctx2(get_mock_io_ctx(io_ctx2)); + + InSequence seq; + + expect_throttler_destroy(); + instance_watcher1->handle_release_leader(); + + // Shutdown instance watcher 1 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx1); + expect_unregister_instance(mock_io_ctx1, 0); + instance_watcher1->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher1; + + // Shutdown instance watcher 2 + expect_release_lock(mock_managed_lock, 0); + expect_unregister_watch(mock_io_ctx2); + expect_unregister_instance(mock_io_ctx2, 0); + instance_watcher2->shut_down(); + + expect_destroy_lock(mock_managed_lock); + delete instance_watcher2; + + TestMockInstanceWatcher::TearDown(); + } + + void expect_throttler_destroy( + std::vector<Context *> *throttler_queue = nullptr) { + EXPECT_CALL(mock_image_sync_throttler, drain(-ESTALE)) + .WillOnce(Invoke([throttler_queue] (int r) { + if (throttler_queue != nullptr) { + for (auto ctx : *throttler_queue) { + ctx->complete(r); + } + } + })); + EXPECT_CALL(mock_image_sync_throttler, destroy()); + } + + void expect_throttler_start_op(const std::string &sync_id, + Context *on_call = nullptr, + Context **on_start_ctx = nullptr) { + EXPECT_CALL(mock_image_sync_throttler, start_op(sync_id, _)) + .WillOnce(Invoke([on_call, on_start_ctx] (const std::string &, + Context *ctx) { + if (on_start_ctx != nullptr) { + *on_start_ctx = ctx; + } else { + ctx->complete(0); + } + if (on_call != nullptr) { + on_call->complete(0); + } + })); + } + + void expect_throttler_finish_op(const std::string &sync_id, + Context *on_finish) { + EXPECT_CALL(mock_image_sync_throttler, finish_op("sync_id")) + .WillOnce(Invoke([on_finish](const std::string &) { + on_finish->complete(0); + })); + } +}; + +TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher1->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + ASSERT_FALSE(instance_watcher1->cancel_sync_request("sync_id")); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher1->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, StartStopOnNonLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, CancelStartedOnNonLeader) { + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + ASSERT_FALSE(instance_watcher2->cancel_sync_request("sync_id")); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, CancelWaitingOnNonLeader) { + InSequence seq; + + C_SaferCond on_start_op_called; + Context *on_start_ctx; + expect_throttler_start_op("sync_id", &on_start_op_called, + &on_start_ctx); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start_op_called.wait()); + + ASSERT_TRUE(instance_watcher2->cancel_sync_request("sync_id")); + // emulate watcher timeout + on_start_ctx->complete(-ETIMEDOUT); + ASSERT_EQ(-ECANCELED, on_start.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, InFlightPrevNotification) { + // start sync when previous notification is still in flight + + InSequence seq; + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start1; + instance_watcher2->notify_sync_request("sync_id", &on_start1); + ASSERT_EQ(0, on_start1.wait()); + + C_SaferCond on_start2; + EXPECT_CALL(mock_image_sync_throttler, finish_op("sync_id")) + .WillOnce(Invoke([this, &on_start2](const std::string &) { + instance_watcher2->notify_sync_request("sync_id", &on_start2); + })); + expect_throttler_start_op("sync_id"); + instance_watcher2->notify_sync_complete("sync_id"); + + ASSERT_EQ(0, on_start2.wait()); + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, NoInFlightReleaseAcquireLeader) { + InSequence seq; + + expect_throttler_destroy(); + instance_watcher1->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnLeaderReleaseLeader) { + InSequence seq; + + expect_throttler_destroy(); + instance_watcher1->handle_release_leader(); + instance_watcher2->handle_acquire_leader(); + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + expect_throttler_destroy(); + instance_watcher2->handle_release_leader(); + instance_watcher2->notify_sync_complete("sync_id"); + + instance_watcher1->handle_acquire_leader(); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnLeaderReleaseLeader) { + InSequence seq; + + C_SaferCond on_start_op_called; + Context *on_start_ctx; + expect_throttler_start_op("sync_id", &on_start_op_called, &on_start_ctx); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start_op_called.wait()); + + std::vector<Context *> throttler_queue = {on_start_ctx}; + expect_throttler_destroy(&throttler_queue); + instance_watcher1->handle_release_leader(); + expect_throttler_start_op("sync_id"); + instance_watcher2->handle_acquire_leader(); + instance_watcher1->handle_update_leader(instance_id2); + + ASSERT_EQ(0, on_start.wait()); + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher1->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); + + expect_throttler_destroy(); + instance_watcher2->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, StartedOnNonLeaderAcquireLeader) { + InSequence seq; + + expect_throttler_destroy(); + instance_watcher1->handle_release_leader(); + instance_watcher2->handle_acquire_leader(); + instance_watcher1->handle_update_leader(instance_id2); + + expect_throttler_start_op("sync_id"); + C_SaferCond on_start; + instance_watcher1->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start.wait()); + + expect_throttler_destroy(); + instance_watcher2->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); + instance_watcher2->handle_update_leader(instance_id1); + + instance_watcher1->notify_sync_complete("sync_id"); +} + +TEST_F(TestMockInstanceWatcher_NotifySync, WaitingOnNonLeaderAcquireLeader) { + InSequence seq; + + C_SaferCond on_start_op_called; + Context *on_start_ctx; + expect_throttler_start_op("sync_id", &on_start_op_called, + &on_start_ctx); + C_SaferCond on_start; + instance_watcher2->notify_sync_request("sync_id", &on_start); + ASSERT_EQ(0, on_start_op_called.wait()); + + std::vector<Context *> throttler_queue = {on_start_ctx}; + expect_throttler_destroy(&throttler_queue); + instance_watcher1->handle_release_leader(); + + EXPECT_CALL(mock_image_sync_throttler, start_op("sync_id", _)) + .WillOnce(WithArg<1>(CompleteContext(0))); + instance_watcher2->handle_acquire_leader(); + instance_watcher1->handle_update_leader(instance_id2); + + ASSERT_EQ(0, on_start.wait()); + + C_SaferCond on_finish; + expect_throttler_finish_op("sync_id", &on_finish); + instance_watcher2->notify_sync_complete("sync_id"); + ASSERT_EQ(0, on_finish.wait()); + + expect_throttler_destroy(); + instance_watcher2->handle_release_leader(); + instance_watcher1->handle_acquire_leader(); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_LeaderWatcher.cc b/src/test/rbd_mirror/test_mock_LeaderWatcher.cc new file mode 100644 index 00000000..8074a5a5 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_LeaderWatcher.cc @@ -0,0 +1,694 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/Utils.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "tools/rbd_mirror/LeaderWatcher.h" +#include "tools/rbd_mirror/Threads.h" + +using librbd::util::create_async_context_callback; + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +struct MockManagedLock { + static MockManagedLock *s_instance; + static MockManagedLock &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + MockManagedLock() { + s_instance = this; + } + + bool m_release_lock_on_shutdown = false; + Context *m_on_released = nullptr; + + MOCK_METHOD0(construct, void()); + MOCK_METHOD0(destroy, void()); + + MOCK_CONST_METHOD0(is_lock_owner, bool()); + + MOCK_METHOD1(shut_down, void(Context *)); + MOCK_METHOD1(try_acquire_lock, void(Context *)); + MOCK_METHOD1(release_lock, void(Context *)); + MOCK_METHOD0(reacquire_lock, void()); + MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *)); + MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *)); + + MOCK_METHOD0(set_state_post_acquiring, void()); + + MOCK_CONST_METHOD0(is_shutdown, bool()); + + MOCK_CONST_METHOD0(is_state_post_acquiring, bool()); + MOCK_CONST_METHOD0(is_state_pre_releasing, bool()); + MOCK_CONST_METHOD0(is_state_locked, bool()); +}; + +MockManagedLock *MockManagedLock::s_instance = nullptr; + +template <> +struct ManagedLock<MockTestImageCtx> { + ManagedLock(librados::IoCtx& ioctx, ContextWQ *work_queue, + const std::string& oid, librbd::Watcher *watcher, + managed_lock::Mode mode, bool blacklist_on_break_lock, + uint32_t blacklist_expire_seconds) + : m_work_queue(work_queue), m_lock("ManagedLock::m_lock") { + MockManagedLock::get_instance().construct(); + } + + virtual ~ManagedLock() { + MockManagedLock::get_instance().destroy(); + } + + ContextWQ *m_work_queue; + + mutable Mutex m_lock; + + bool is_lock_owner() const { + return MockManagedLock::get_instance().is_lock_owner(); + } + + void shut_down(Context *on_shutdown) { + if (MockManagedLock::get_instance().m_release_lock_on_shutdown) { + on_shutdown = new FunctionContext( + [this, on_shutdown](int r) { + MockManagedLock::get_instance().m_release_lock_on_shutdown = false; + shut_down(on_shutdown); + }); + release_lock(on_shutdown); + return; + } + + MockManagedLock::get_instance().shut_down(on_shutdown); + } + + void try_acquire_lock(Context *on_acquired) { + Context *post_acquire_ctx = create_async_context_callback( + m_work_queue, new FunctionContext( + [this, on_acquired](int r) { + post_acquire_lock_handler(r, on_acquired); + })); + MockManagedLock::get_instance().try_acquire_lock(post_acquire_ctx); + } + + void release_lock(Context *on_released) { + ceph_assert(MockManagedLock::get_instance().m_on_released == nullptr); + MockManagedLock::get_instance().m_on_released = on_released; + + Context *post_release_ctx = new FunctionContext( + [this](int r) { + ceph_assert(MockManagedLock::get_instance().m_on_released != nullptr); + post_release_lock_handler(false, r, + MockManagedLock::get_instance().m_on_released); + MockManagedLock::get_instance().m_on_released = nullptr; + }); + + Context *release_ctx = new FunctionContext( + [this, post_release_ctx](int r) { + if (r < 0) { + MockManagedLock::get_instance().m_on_released->complete(r); + } else { + MockManagedLock::get_instance().release_lock(post_release_ctx); + } + }); + + Context *pre_release_ctx = new FunctionContext( + [this, release_ctx](int r) { + bool shutting_down = + MockManagedLock::get_instance().m_release_lock_on_shutdown; + pre_release_lock_handler(shutting_down, release_ctx); + }); + + m_work_queue->queue(pre_release_ctx, 0); + } + + void reacquire_lock(Context* on_finish) { + MockManagedLock::get_instance().reacquire_lock(); + } + + void get_locker(managed_lock::Locker *locker, Context *on_finish) { + MockManagedLock::get_instance().get_locker(locker, on_finish); + } + + void break_lock(const managed_lock::Locker &locker, bool force_break_lock, + Context *on_finish) { + MockManagedLock::get_instance().break_lock(locker, force_break_lock, + on_finish); + } + + void set_state_post_acquiring() { + MockManagedLock::get_instance().set_state_post_acquiring(); + } + + bool is_shutdown() const { + return MockManagedLock::get_instance().is_shutdown(); + } + + bool is_state_post_acquiring() const { + return MockManagedLock::get_instance().is_state_post_acquiring(); + } + + bool is_state_pre_releasing() const { + return MockManagedLock::get_instance().is_state_pre_releasing(); + } + + bool is_state_locked() const { + return MockManagedLock::get_instance().is_state_locked(); + } + + virtual void post_acquire_lock_handler(int r, Context *on_finish) = 0; + virtual void pre_release_lock_handler(bool shutting_down, + Context *on_finish) = 0; + virtual void post_release_lock_handler(bool shutting_down, int r, + Context *on_finish) = 0; +}; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + Mutex &timer_lock; + SafeTimer *timer; + ContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer_lock(threads->timer_lock), timer(threads->timer), + work_queue(threads->work_queue) { + } +}; + +template <> +struct MirrorStatusWatcher<librbd::MockTestImageCtx> { + static MirrorStatusWatcher* s_instance; + + static MirrorStatusWatcher *create(librados::IoCtx &io_ctx, + ContextWQ *work_queue) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MirrorStatusWatcher() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~MirrorStatusWatcher() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD1(init, void(Context *)); + MOCK_METHOD1(shut_down, void(Context *)); +}; + +MirrorStatusWatcher<librbd::MockTestImageCtx> *MirrorStatusWatcher<librbd::MockTestImageCtx>::s_instance = nullptr; + +template <> +struct Instances<librbd::MockTestImageCtx> { + static Instances* s_instance; + + static Instances *create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &ioctx, + const std::string& instance_id, + instances::Listener&) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + Instances() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~Instances() { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD0(destroy, void()); + MOCK_METHOD1(init, void(Context *)); + MOCK_METHOD1(shut_down, void(Context *)); + MOCK_METHOD1(acked, void(const std::vector<std::string> &)); + MOCK_METHOD0(unblock_listener, void()); +}; + +Instances<librbd::MockTestImageCtx> *Instances<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace mirror +} // namespace rbd + + +// template definitions +#include "tools/rbd_mirror/LeaderWatcher.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; + +using librbd::MockManagedLock; + +struct MockListener : public leader_watcher::Listener { + static MockListener* s_instance; + + MockListener() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + + ~MockListener() override { + ceph_assert(s_instance == this); + s_instance = nullptr; + } + + MOCK_METHOD1(post_acquire_handler, void(Context *)); + MOCK_METHOD1(pre_release_handler, void(Context *)); + + MOCK_METHOD1(update_leader_handler, void(const std::string &)); + MOCK_METHOD1(handle_instances_added, void(const InstanceIds&)); + MOCK_METHOD1(handle_instances_removed, void(const InstanceIds&)); +}; + +MockListener *MockListener::s_instance = nullptr; + +class TestMockLeaderWatcher : public TestMockFixture { +public: + typedef MirrorStatusWatcher<librbd::MockTestImageCtx> MockMirrorStatusWatcher; + typedef Instances<librbd::MockTestImageCtx> MockInstances; + typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + void SetUp() override { + TestMockFixture::SetUp(); + m_mock_threads = new MockThreads(m_threads); + } + + void TearDown() override { + delete m_mock_threads; + TestMockFixture::TearDown(); + } + + void expect_construct(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, construct()); + } + + void expect_destroy(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, destroy()); + } + + void expect_is_lock_owner(MockManagedLock &mock_managed_lock, bool owner) { + EXPECT_CALL(mock_managed_lock, is_lock_owner()) + .WillOnce(Return(owner)); + } + + void expect_shut_down(MockManagedLock &mock_managed_lock, + bool release_lock_on_shutdown, int r) { + mock_managed_lock.m_release_lock_on_shutdown = release_lock_on_shutdown; + EXPECT_CALL(mock_managed_lock, shut_down(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_try_acquire_lock(MockManagedLock &mock_managed_lock, int r) { + EXPECT_CALL(mock_managed_lock, try_acquire_lock(_)) + .WillOnce(CompleteContext(r)); + if (r == 0) { + expect_set_state_post_acquiring(mock_managed_lock); + } + } + + void expect_release_lock(MockManagedLock &mock_managed_lock, int r, + Context *on_finish = nullptr) { + EXPECT_CALL(mock_managed_lock, release_lock(_)) + .WillOnce(Invoke([on_finish, &mock_managed_lock, r](Context *ctx) { + if (on_finish != nullptr) { + auto on_released = mock_managed_lock.m_on_released; + ceph_assert(on_released != nullptr); + mock_managed_lock.m_on_released = new FunctionContext( + [on_released, on_finish](int r) { + on_released->complete(r); + on_finish->complete(r); + }); + } + ctx->complete(r); + })); + } + + void expect_get_locker(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r) { + EXPECT_CALL(mock_managed_lock, get_locker(_, _)) + .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out, + Context *ctx) { + if (r == 0) { + *out = locker; + } + ctx->complete(r); + })); + } + + void expect_break_lock(MockManagedLock &mock_managed_lock, + const librbd::managed_lock::Locker &locker, int r, + Context *on_finish) { + EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _)) + .WillOnce(Invoke([on_finish, r](const librbd::managed_lock::Locker &, + bool, Context *ctx) { + ctx->complete(r); + on_finish->complete(0); + })); + } + + void expect_set_state_post_acquiring(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, set_state_post_acquiring()); + } + + void expect_is_shutdown(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, is_shutdown()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + } + + void expect_is_leader(MockManagedLock &mock_managed_lock, bool post_acquiring, + bool locked) { + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .WillOnce(Return(post_acquiring)); + if (!post_acquiring) { + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .WillOnce(Return(locked)); + } + } + + void expect_is_leader(MockManagedLock &mock_managed_lock) { + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_pre_releasing()) + .Times(AtLeast(0)).WillRepeatedly(Return(false)); + } + + void expect_notify_heartbeat(MockManagedLock &mock_managed_lock, + Context *on_finish) { + // is_leader in notify_heartbeat + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .WillOnce(Return(true)); + + // is_leader in handle_notify_heartbeat + EXPECT_CALL(mock_managed_lock, is_state_post_acquiring()) + .WillOnce(Return(false)); + EXPECT_CALL(mock_managed_lock, is_state_locked()) + .WillOnce(DoAll(Invoke([on_finish]() { + on_finish->complete(0); + }), + Return(true))); + } + + void expect_destroy(MockMirrorStatusWatcher &mock_mirror_status_watcher) { + EXPECT_CALL(mock_mirror_status_watcher, destroy()); + } + + void expect_init(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) { + EXPECT_CALL(mock_mirror_status_watcher, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_shut_down(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) { + EXPECT_CALL(mock_mirror_status_watcher, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + expect_destroy(mock_mirror_status_watcher); + } + + void expect_destroy(MockInstances &mock_instances) { + EXPECT_CALL(mock_instances, destroy()); + } + + void expect_init(MockInstances &mock_instances, int r) { + EXPECT_CALL(mock_instances, init(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + } + + void expect_shut_down(MockInstances &mock_instances, int r) { + EXPECT_CALL(mock_instances, shut_down(_)) + .WillOnce(CompleteContext(m_mock_threads->work_queue, r)); + expect_destroy(mock_instances); + } + + void expect_acquire_notify(MockManagedLock &mock_managed_lock, + MockListener &mock_listener, int r) { + expect_is_leader(mock_managed_lock, true, false); + EXPECT_CALL(mock_listener, post_acquire_handler(_)) + .WillOnce(CompleteContext(r)); + expect_is_leader(mock_managed_lock, true, false); + } + + void expect_release_notify(MockManagedLock &mock_managed_lock, + MockListener &mock_listener, int r) { + expect_is_leader(mock_managed_lock, false, false); + EXPECT_CALL(mock_listener, pre_release_handler(_)) + .WillOnce(CompleteContext(r)); + expect_is_leader(mock_managed_lock, false, false); + } + + void expect_unblock_listener(MockInstances& mock_instances) { + EXPECT_CALL(mock_instances, unblock_listener()); + } + + void expect_instances_acked(MockInstances& mock_instances) { + EXPECT_CALL(mock_instances, acked(_)); + } + + MockThreads *m_mock_threads; +}; + +TEST_F(TestMockLeaderWatcher, InitShutdown) { + MockManagedLock mock_managed_lock; + MockMirrorStatusWatcher mock_mirror_status_watcher; + MockInstances mock_instances; + MockListener listener; + + expect_is_shutdown(mock_managed_lock); + expect_destroy(mock_managed_lock); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + expect_init(mock_mirror_status_watcher, 0); + C_SaferCond on_heartbeat_finish; + expect_is_leader(mock_managed_lock, false, false); + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Shutdown + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + expect_release_lock(mock_managed_lock, 0); + expect_shut_down(mock_managed_lock, true, 0); + expect_shut_down(mock_mirror_status_watcher, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +TEST_F(TestMockLeaderWatcher, InitReleaseShutdown) { + MockManagedLock mock_managed_lock; + MockMirrorStatusWatcher mock_mirror_status_watcher; + MockInstances mock_instances; + MockListener listener; + + expect_is_shutdown(mock_managed_lock); + expect_destroy(mock_managed_lock); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + expect_init(mock_mirror_status_watcher, 0); + C_SaferCond on_heartbeat_finish; + expect_is_leader(mock_managed_lock, false, false); + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Release + expect_is_leader(mock_managed_lock, false, true); + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + C_SaferCond on_release; + expect_release_lock(mock_managed_lock, 0, &on_release); + + leader_watcher.release_leader(); + ASSERT_EQ(0, on_release.wait()); + + // Shutdown + expect_shut_down(mock_managed_lock, false, 0); + expect_shut_down(mock_mirror_status_watcher, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +TEST_F(TestMockLeaderWatcher, InitStatusWatcherError) { + MockManagedLock mock_managed_lock; + MockMirrorStatusWatcher mock_mirror_status_watcher; + MockInstances mock_instances; + MockListener listener; + + expect_is_shutdown(mock_managed_lock); + expect_is_leader(mock_managed_lock); + expect_destroy(mock_managed_lock); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + expect_init(mock_mirror_status_watcher, -EINVAL); + ASSERT_EQ(-EINVAL, leader_watcher.init()); + + // Shutdown + expect_shut_down(mock_managed_lock, false, 0); + expect_shut_down(mock_mirror_status_watcher, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +TEST_F(TestMockLeaderWatcher, AcquireError) { + MockManagedLock mock_managed_lock; + MockMirrorStatusWatcher mock_mirror_status_watcher; + MockInstances mock_instances; + MockListener listener; + + expect_is_shutdown(mock_managed_lock); + expect_is_leader(mock_managed_lock); + expect_destroy(mock_managed_lock); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + expect_init(mock_mirror_status_watcher, 0); + C_SaferCond on_heartbeat_finish; + expect_is_leader(mock_managed_lock, false, false); + expect_try_acquire_lock(mock_managed_lock, -EAGAIN); + expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT); + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Shutdown + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + expect_release_lock(mock_managed_lock, 0); + expect_shut_down(mock_managed_lock, true, 0); + expect_shut_down(mock_mirror_status_watcher, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +TEST_F(TestMockLeaderWatcher, Break) { + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", "1")); + EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_missed_heartbeats", + "1")); + CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct()); + int max_acquire_attempts = cct->_conf.get_val<uint64_t>( + "rbd_mirror_leader_max_acquire_attempts_before_break"); + + MockManagedLock mock_managed_lock; + MockMirrorStatusWatcher mock_mirror_status_watcher; + MockInstances mock_instances; + MockListener listener; + librbd::managed_lock::Locker + locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123}; + + expect_is_shutdown(mock_managed_lock); + expect_is_leader(mock_managed_lock); + expect_destroy(mock_managed_lock); + EXPECT_CALL(listener, update_leader_handler(_)); + + InSequence seq; + + expect_construct(mock_managed_lock); + MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener); + + // Init + expect_init(mock_mirror_status_watcher, 0); + expect_is_leader(mock_managed_lock, false, false); + for (int i = 0; i < max_acquire_attempts; i++) { + expect_try_acquire_lock(mock_managed_lock, -EAGAIN); + expect_get_locker(mock_managed_lock, locker, 0); + } + C_SaferCond on_break; + expect_break_lock(mock_managed_lock, locker, 0, &on_break); + C_SaferCond on_heartbeat_finish; + expect_try_acquire_lock(mock_managed_lock, 0); + expect_init(mock_instances, 0); + expect_acquire_notify(mock_managed_lock, listener, 0); + expect_unblock_listener(mock_instances); + expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish); + expect_instances_acked(mock_instances); + + ASSERT_EQ(0, leader_watcher.init()); + ASSERT_EQ(0, on_heartbeat_finish.wait()); + + // Shutdown + expect_release_notify(mock_managed_lock, listener, 0); + expect_shut_down(mock_instances, 0); + expect_release_lock(mock_managed_lock, 0); + expect_shut_down(mock_managed_lock, true, 0); + expect_shut_down(mock_mirror_status_watcher, 0); + expect_is_leader(mock_managed_lock, false, false); + + leader_watcher.shut_down(); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc new file mode 100644 index 00000000..e6bc9b1e --- /dev/null +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -0,0 +1,468 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/api/Config.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librados_test_stub/MockTestMemCluster.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "tools/rbd_mirror/PoolReplayer.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/ImageMap.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/InstanceReplayer.h" +#include "tools/rbd_mirror/LeaderWatcher.h" +#include "tools/rbd_mirror/PoolWatcher.h" +#include "tools/rbd_mirror/ServiceDaemon.h" +#include "tools/rbd_mirror/Threads.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +namespace api { + +template <> +class Config<MockTestImageCtx> { +public: + static void apply_pool_overrides(librados::IoCtx& io_ctx, + ConfigProxy* config_proxy) { + } +}; + +} + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct ImageDeleter<librbd::MockTestImageCtx> { + static ImageDeleter* s_instance; + + static ImageDeleter* create(librados::IoCtx &ioctx, + Threads<librbd::MockTestImageCtx> *threads, + ServiceDaemon<librbd::MockTestImageCtx> *service_daemon) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*)); + + ImageDeleter() { + s_instance = this; + } +}; + +ImageDeleter<librbd::MockTestImageCtx>* ImageDeleter<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct ImageMap<librbd::MockTestImageCtx> { + static ImageMap* s_instance; + + static ImageMap *create(librados::IoCtx &ioctx, + Threads<librbd::MockTestImageCtx> *threads, + const std::string& instance_id, + image_map::Listener &listener) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + MOCK_METHOD1(update_instances_added, void(const std::vector<std::string>&)); + MOCK_METHOD1(update_instances_removed, void(const std::vector<std::string>&)); + + MOCK_METHOD3(update_images_mock, void(const std::string&, + const std::set<std::string>&, + const std::set<std::string>&)); + void update_images(const std::string& mirror_uuid, + std::set<std::string>&& added, + std::set<std::string>&& removed) { + update_images_mock(mirror_uuid, added, removed); + } + + ImageMap() { + s_instance = this; + } +}; + +ImageMap<librbd::MockTestImageCtx>* ImageMap<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct InstanceReplayer<librbd::MockTestImageCtx> { + static InstanceReplayer* s_instance; + + static InstanceReplayer* create(Threads<librbd::MockTestImageCtx> *threads, + ServiceDaemon<librbd::MockTestImageCtx> *service_daemon, + RadosRef rados, const std::string& uuid, + int64_t pool_id) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD0(is_blacklisted, bool()); + + MOCK_METHOD0(start, void()); + MOCK_METHOD0(stop, void()); + MOCK_METHOD0(restart, void()); + MOCK_METHOD0(flush, void()); + + MOCK_METHOD2(print_status, void(Formatter*, std::stringstream*)); + + MOCK_METHOD2(add_peer, void(const std::string&, librados::IoCtx&)); + + MOCK_METHOD0(init, void()); + MOCK_METHOD0(shut_down, void()); + MOCK_METHOD1(release_all, void(Context*)); + + InstanceReplayer() { + s_instance = this; + } +}; + +InstanceReplayer<librbd::MockTestImageCtx>* InstanceReplayer<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct InstanceWatcher<librbd::MockTestImageCtx> { + static InstanceWatcher* s_instance; + + static InstanceWatcher* create(librados::IoCtx &ioctx, + MockContextWQ* work_queue, + InstanceReplayer<librbd::MockTestImageCtx>* instance_replayer) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD0(handle_acquire_leader, void()); + MOCK_METHOD0(handle_release_leader, void()); + + MOCK_METHOD0(get_instance_id, std::string()); + + MOCK_METHOD2(print_sync_status, void(Formatter*, std::stringstream*)); + + MOCK_METHOD0(init, int()); + MOCK_METHOD0(shut_down, void()); + + MOCK_METHOD3(notify_image_acquire, void(const std::string&, + const std::string&, + Context*)); + MOCK_METHOD3(notify_image_release, void(const std::string&, + const std::string&, + Context*)); + MOCK_METHOD4(notify_peer_image_removed, void(const std::string&, + const std::string&, + const std::string&, + Context*)); + + MOCK_METHOD1(handle_update_leader, void(const std::string&)); + + InstanceWatcher() { + s_instance = this; + } + +}; + +InstanceWatcher<librbd::MockTestImageCtx>* InstanceWatcher<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct LeaderWatcher<librbd::MockTestImageCtx> { + static LeaderWatcher* s_instance; + + static LeaderWatcher *create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &ioctx, + leader_watcher::Listener* listener) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD0(is_blacklisted, bool()); + MOCK_METHOD0(is_leader, bool()); + MOCK_METHOD0(release_leader, void()); + + MOCK_METHOD1(get_leader_instance_id, void(std::string*)); + MOCK_METHOD1(list_instances, void(std::vector<std::string>*)); + + MOCK_METHOD0(init, int()); + MOCK_METHOD0(shut_down, int()); + + LeaderWatcher() { + s_instance = this; + } + +}; + +LeaderWatcher<librbd::MockTestImageCtx>* LeaderWatcher<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct PoolWatcher<librbd::MockTestImageCtx> { + static PoolWatcher* s_instance; + + static PoolWatcher *create(Threads<librbd::MockTestImageCtx> *threads, + librados::IoCtx &ioctx, + pool_watcher::Listener& listener) { + ceph_assert(s_instance != nullptr); + return s_instance; + } + + MOCK_METHOD0(is_blacklisted, bool()); + + MOCK_METHOD0(get_image_count, uint64_t()); + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + PoolWatcher() { + s_instance = this; + } + +}; + +PoolWatcher<librbd::MockTestImageCtx>* PoolWatcher<librbd::MockTestImageCtx>::s_instance = nullptr; + +template<> +struct ServiceDaemon<librbd::MockTestImageCtx> { + MOCK_METHOD3(add_or_update_attribute, + void(int64_t, const std::string&, + const service_daemon::AttributeValue&)); + MOCK_METHOD2(remove_attribute, + void(int64_t, const std::string&)); + + MOCK_METHOD4(add_or_update_callout, uint64_t(int64_t, uint64_t, + service_daemon::CalloutLevel, + const std::string&)); + MOCK_METHOD2(remove_callout, void(int64_t, uint64_t)); +}; + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + Mutex &timer_lock; + Cond timer_cond; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/PoolReplayer.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockPoolReplayer : public TestMockFixture { +public: + typedef PoolReplayer<librbd::MockTestImageCtx> MockPoolReplayer; + typedef ImageMap<librbd::MockTestImageCtx> MockImageMap; + typedef InstanceReplayer<librbd::MockTestImageCtx> MockInstanceReplayer; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; + typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher; + typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher; + typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + + void expect_connect(librados::MockTestMemCluster& mock_cluster, + librados::MockTestMemRadosClient* mock_rados_client, + const std::string& cluster_name, CephContext** cct_ref) { + EXPECT_CALL(mock_cluster, create_rados_client(_)) + .WillOnce(Invoke([cluster_name, mock_rados_client, cct_ref](CephContext* cct) { + EXPECT_EQ(cluster_name, cct->_conf->cluster); + if (cct_ref != nullptr) { + cct->get(); + *cct_ref = cct; + } + + return mock_rados_client; + })); + } + + void expect_create_ioctx(librados::MockTestMemRadosClient* mock_rados_client, + librados::MockTestMemIoCtxImpl* mock_io_ctx_impl) { + EXPECT_CALL(*mock_rados_client, create_ioctx(_, _)) + .WillOnce(Invoke([mock_io_ctx_impl](int64_t id, const std::string& name) { + return mock_io_ctx_impl; + })); + } + + void expect_mirror_uuid_get(librados::MockTestMemIoCtxImpl *io_ctx_impl, + const std::string &uuid, int r) { + bufferlist out_bl; + encode(uuid, out_bl); + + EXPECT_CALL(*io_ctx_impl, + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), + _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + + void expect_instance_replayer_is_blacklisted( + MockInstanceReplayer &mock_instance_replayer, bool blacklisted) { + EXPECT_CALL(mock_instance_replayer, is_blacklisted()) + .WillRepeatedly(Return(blacklisted)); + } + + void expect_instance_replayer_init(MockInstanceReplayer& mock_instance_replayer) { + EXPECT_CALL(mock_instance_replayer, init()); + } + + void expect_instance_replayer_shut_down(MockInstanceReplayer& mock_instance_replayer) { + EXPECT_CALL(mock_instance_replayer, shut_down()); + } + + void expect_instance_replayer_stop(MockInstanceReplayer& mock_instance_replayer) { + EXPECT_CALL(mock_instance_replayer, stop()); + } + + void expect_instance_replayer_add_peer(MockInstanceReplayer& mock_instance_replayer, + const std::string& uuid) { + EXPECT_CALL(mock_instance_replayer, add_peer(uuid, _)); + } + + void expect_instance_watcher_get_instance_id( + MockInstanceWatcher& mock_instance_watcher, + const std::string &instance_id) { + EXPECT_CALL(mock_instance_watcher, get_instance_id()) + .WillOnce(Return(instance_id)); + } + + void expect_instance_watcher_init(MockInstanceWatcher& mock_instance_watcher, + int r) { + EXPECT_CALL(mock_instance_watcher, init()) + .WillOnce(Return(r)); + } + + void expect_instance_watcher_shut_down(MockInstanceWatcher& mock_instance_watcher) { + EXPECT_CALL(mock_instance_watcher, shut_down()); + } + + void expect_leader_watcher_is_blacklisted( + MockLeaderWatcher &mock_leader_watcher, bool blacklisted) { + EXPECT_CALL(mock_leader_watcher, is_blacklisted()) + .WillRepeatedly(Return(blacklisted)); + } + + void expect_leader_watcher_init(MockLeaderWatcher& mock_leader_watcher, + int r) { + EXPECT_CALL(mock_leader_watcher, init()) + .WillOnce(Return(r)); + } + + void expect_leader_watcher_shut_down(MockLeaderWatcher& mock_leader_watcher) { + EXPECT_CALL(mock_leader_watcher, shut_down()); + } + + void expect_service_daemon_add_or_update_attribute( + MockServiceDaemon &mock_service_daemon, const std::string& key, + const service_daemon::AttributeValue& value) { + EXPECT_CALL(mock_service_daemon, add_or_update_attribute(_, _, _)); + } + + void expect_service_daemon_add_or_update_instance_id_attribute( + MockInstanceWatcher& mock_instance_watcher, + MockServiceDaemon &mock_service_daemon) { + expect_instance_watcher_get_instance_id(mock_instance_watcher, "1234"); + expect_service_daemon_add_or_update_attribute(mock_service_daemon, + "instance_id", "1234"); + } +}; + +TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { + PeerSpec peer_spec{"uuid", "cluster name", "client.name"}; + peer_spec.mon_host = "123"; + peer_spec.key = "234"; + + auto mock_instance_replayer = new MockInstanceReplayer(); + expect_instance_replayer_is_blacklisted(*mock_instance_replayer, false); + + auto mock_leader_watcher = new MockLeaderWatcher(); + expect_leader_watcher_is_blacklisted(*mock_leader_watcher, false); + + InSequence seq; + + auto& mock_cluster = get_mock_cluster(); + auto mock_local_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + expect_connect(mock_cluster, mock_local_rados_client, "ceph", nullptr); + + auto mock_remote_rados_client = mock_cluster.do_create_rados_client( + g_ceph_context); + CephContext* remote_cct = nullptr; + expect_connect(mock_cluster, mock_remote_rados_client, "cluster name", + &remote_cct); + + auto mock_local_io_ctx = mock_local_rados_client->do_create_ioctx( + m_local_io_ctx.get_id(), m_local_io_ctx.get_pool_name()); + expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); + + expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + + expect_instance_replayer_init(*mock_instance_replayer); + expect_instance_replayer_add_peer(*mock_instance_replayer, "uuid"); + + auto mock_instance_watcher = new MockInstanceWatcher(); + expect_instance_watcher_init(*mock_instance_watcher, 0); + + MockServiceDaemon mock_service_daemon; + expect_service_daemon_add_or_update_instance_id_attribute( + *mock_instance_watcher, mock_service_daemon); + + expect_leader_watcher_init(*mock_leader_watcher, 0); + + MockThreads mock_threads(m_threads); + MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, + m_local_io_ctx.get_id(), peer_spec, {}); + pool_replayer.init(); + + ASSERT_TRUE(remote_cct != nullptr); + ASSERT_EQ("123", remote_cct->_conf.get_val<std::string>("mon_host")); + ASSERT_EQ("234", remote_cct->_conf.get_val<std::string>("key")); + remote_cct->put(); + + expect_instance_replayer_stop(*mock_instance_replayer); + expect_leader_watcher_shut_down(*mock_leader_watcher); + expect_instance_watcher_shut_down(*mock_instance_watcher); + expect_instance_replayer_shut_down(*mock_instance_replayer); + + pool_replayer.shut_down(); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_PoolWatcher.cc b/src/test/rbd_mirror/test_mock_PoolWatcher.cc new file mode 100644 index 00000000..b4dd66e8 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_PoolWatcher.cc @@ -0,0 +1,897 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "librbd/MirroringWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/PoolWatcher.h" +#include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h" +#include "include/stringify.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +struct MockMirroringWatcher { + static MockMirroringWatcher *s_instance; + static MockMirroringWatcher &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + MockMirroringWatcher() { + s_instance = this; + } + + MOCK_CONST_METHOD0(is_unregistered, bool()); + MOCK_METHOD1(register_watch, void(Context*)); + MOCK_METHOD1(unregister_watch, void(Context*)); + + MOCK_CONST_METHOD0(get_oid, std::string()); +}; + +template <> +struct MirroringWatcher<MockTestImageCtx> { + static MirroringWatcher *s_instance; + + MirroringWatcher(librados::IoCtx &io_ctx, ::MockContextWQ *work_queue) { + s_instance = this; + } + virtual ~MirroringWatcher() { + } + + static MirroringWatcher<MockTestImageCtx> &get_instance() { + ceph_assert(s_instance != nullptr); + return *s_instance; + } + + virtual void handle_rewatch_complete(int r) = 0; + + virtual void handle_mode_updated(cls::rbd::MirrorMode mirror_mode) = 0; + virtual void handle_image_updated(cls::rbd::MirrorImageState state, + const std::string &remote_image_id, + const std::string &global_image_id) = 0; + + bool is_unregistered() const { + return MockMirroringWatcher::get_instance().is_unregistered(); + } + void register_watch(Context *ctx) { + MockMirroringWatcher::get_instance().register_watch(ctx); + } + void unregister_watch(Context *ctx) { + MockMirroringWatcher::get_instance().unregister_watch(ctx); + } + std::string get_oid() const { + return MockMirroringWatcher::get_instance().get_oid(); + } +}; + +MockMirroringWatcher *MockMirroringWatcher::s_instance = nullptr; +MirroringWatcher<MockTestImageCtx> *MirroringWatcher<MockTestImageCtx>::s_instance = nullptr; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads<librbd::MockTestImageCtx> { + MockSafeTimer *timer; + Mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads<librbd::ImageCtx> *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +namespace pool_watcher { + +template <> +struct RefreshImagesRequest<librbd::MockTestImageCtx> { + ImageIds *image_ids = nullptr; + Context *on_finish = nullptr; + static RefreshImagesRequest *s_instance; + static RefreshImagesRequest *create(librados::IoCtx &io_ctx, + ImageIds *image_ids, + Context *on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->image_ids = image_ids; + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + RefreshImagesRequest() { + s_instance = this; + } +}; + +RefreshImagesRequest<librbd::MockTestImageCtx> *RefreshImagesRequest<librbd::MockTestImageCtx>::s_instance = nullptr; + +} // namespace pool_watcher + +} // namespace mirror +} // namespace rbd + +// template definitions +#include "tools/rbd_mirror/PoolWatcher.cc" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithoutArgs; + +class TestMockPoolWatcher : public TestMockFixture { +public: + typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher; + typedef Threads<librbd::MockTestImageCtx> MockThreads; + typedef pool_watcher::RefreshImagesRequest<librbd::MockTestImageCtx> MockRefreshImagesRequest; + typedef librbd::MockMirroringWatcher MockMirroringWatcher; + typedef librbd::MirroringWatcher<librbd::MockTestImageCtx> MirroringWatcher; + + struct MockListener : pool_watcher::Listener { + TestMockPoolWatcher *test; + + MockListener(TestMockPoolWatcher *test) : test(test) { + } + + MOCK_METHOD3(mock_handle_update, void(const std::string &, const ImageIds &, + const ImageIds &)); + void handle_update(const std::string &mirror_uuid, + ImageIds &&added_image_ids, + ImageIds &&removed_image_ids) override { + mock_handle_update(mirror_uuid, added_image_ids, removed_image_ids); + } + }; + + TestMockPoolWatcher() : m_lock("TestMockPoolWatcher::m_lock") { + } + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_mirroring_watcher_is_unregistered(MockMirroringWatcher &mock_mirroring_watcher, + bool unregistered) { + EXPECT_CALL(mock_mirroring_watcher, is_unregistered()) + .WillOnce(Return(unregistered)); + } + + void expect_mirroring_watcher_register(MockMirroringWatcher &mock_mirroring_watcher, + int r) { + EXPECT_CALL(mock_mirroring_watcher, register_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_mirroring_watcher_unregister(MockMirroringWatcher &mock_mirroring_watcher, + int r) { + EXPECT_CALL(mock_mirroring_watcher, unregister_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_refresh_images(MockRefreshImagesRequest &request, + const ImageIds &image_ids, int r) { + EXPECT_CALL(request, send()) + .WillOnce(Invoke([&request, image_ids, r]() { + *request.image_ids = image_ids; + request.on_finish->complete(r); + })); + } + + void expect_listener_handle_update(MockListener &mock_listener, + const std::string &mirror_uuid, + const ImageIds &added_image_ids, + const ImageIds &removed_image_ids) { + EXPECT_CALL(mock_listener, mock_handle_update(mirror_uuid, added_image_ids, + removed_image_ids)) + .WillOnce(WithoutArgs(Invoke([this]() { + Mutex::Locker locker(m_lock); + ++m_update_count; + m_cond.Signal(); + }))); + } + + void expect_mirror_uuid_get(librados::IoCtx &io_ctx, + const std::string &uuid, int r) { + bufferlist out_bl; + encode(uuid, out_bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"), + _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + + void expect_timer_add_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = + new FunctionContext([this, ctx](int r) { + Mutex::Locker timer_locker(m_threads->timer_lock); + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), + ReturnArg<1>())); + } + + int when_shut_down(MockPoolWatcher &mock_pool_watcher) { + C_SaferCond ctx; + mock_pool_watcher.shut_down(&ctx); + return ctx.wait(); + } + + bool wait_for_update(uint32_t count) { + Mutex::Locker locker(m_lock); + while (m_update_count < count) { + if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) { + break; + } + } + if (m_update_count < count) { + return false; + } + + m_update_count -= count; + return true; + } + + Mutex m_lock; + Cond m_cond; + uint32_t m_update_count = 0; +}; + +TEST_F(TestMockPoolWatcher, EmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, NonEmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1"}, + {"global id 2", "remote id 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1"}, + {"global id 2", "remote id 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + bool refresh_sent = false; + EXPECT_CALL(mock_refresh_images_request, send()) + .WillOnce(Invoke([this, &mock_refresh_images_request, &image_ids, + &refresh_sent]() { + *mock_refresh_images_request.image_ids = image_ids; + + Mutex::Locker locker(m_lock); + refresh_sent = true; + m_cond.Signal(); + })); + + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + image_ids = { + {"global id 1", "remote id 1a"}, + {"global id 3", "remote id 3"}}; + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + mock_pool_watcher.init(nullptr); + + { + Mutex::Locker locker(m_lock); + while (!refresh_sent) { + m_cond.Wait(m_lock); + } + } + + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3"); + + mock_refresh_images_request.on_finish->complete(0); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, Notify) { + MockThreads mock_threads(m_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1"}, + {"global id 2", "remote id 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillOnce(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + Context *notify_ctx = nullptr; + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillOnce(Invoke([this, ¬ify_ctx](Context *ctx, int r) { + Mutex::Locker locker(m_lock); + ASSERT_EQ(nullptr, notify_ctx); + notify_ctx = ctx; + m_cond.Signal(); + })); + expect_listener_handle_update( + mock_listener, "remote uuid", + {{"global id 1", "remote id 1a"}, {"global id 3", "remote id 3"}}, + {{"global id 1", "remote id 1"}, {"global id 2", "remote id 2"}}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + C_SaferCond flush_ctx; + m_threads->work_queue->queue(&flush_ctx, 0); + ASSERT_EQ(0, flush_ctx.wait()); + + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_DISABLED, "remote id 2", "global id 2"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1"); + MirroringWatcher::get_instance().handle_image_updated( + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3"); + notify_ctx->complete(0); + + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RegisterWatcherBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, -EBLACKLISTED); + + MockListener mock_listener(this); + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + ASSERT_TRUE(mock_pool_watcher.is_blacklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RegisterWatcherMissing) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, -ENOENT); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RegisterWatcherError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, -EINVAL); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RefreshBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, -EBLACKLISTED); + + MockListener mock_listener(this); + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + ASSERT_TRUE(mock_pool_watcher.is_blacklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RefreshMissing) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, -ENOENT); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RefreshError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, -EINVAL); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, GetMirrorUuidBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EBLACKLISTED); + + MockListener mock_listener(this); + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + ASSERT_TRUE(mock_pool_watcher.is_blacklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, GetMirrorUuidMissing) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "", -ENOENT); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(-ENOENT, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, GetMirrorUuidError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EINVAL); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, Rewatch) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + expect_timer_add_event(mock_threads); + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + expect_listener_handle_update(mock_listener, "remote uuid", + {{"global id", "image id"}}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + MirroringWatcher::get_instance().handle_rewatch_complete(0); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RewatchBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + MirroringWatcher::get_instance().handle_rewatch_complete(-EBLACKLISTED); + ASSERT_TRUE(mock_pool_watcher.is_blacklisted()); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, RewatchError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + expect_timer_add_event(mock_threads); + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + expect_listener_handle_update(mock_listener, "remote uuid", + {{"global id", "image id"}}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + MirroringWatcher::get_instance().handle_rewatch_complete(-EINVAL); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, DeferredRefresh) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + MockRefreshImagesRequest mock_refresh_images_request; + + EXPECT_CALL(mock_refresh_images_request, send()) + .WillOnce(Invoke([&mock_refresh_images_request]() { + *mock_refresh_images_request.image_ids = {}; + MirroringWatcher::get_instance().handle_rewatch_complete(0); + mock_refresh_images_request.on_finish->complete(0); + })); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + expect_timer_add_event(mock_threads); + + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, {}, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", {}, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + ASSERT_TRUE(wait_for_update(1)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +TEST_F(TestMockPoolWatcher, MirrorUuidUpdated) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + MockMirroringWatcher mock_mirroring_watcher; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true); + expect_mirroring_watcher_register(mock_mirroring_watcher, 0); + + ImageIds image_ids{ + {"global id 1", "remote id 1"}, + {"global id 2", "remote id 2"}}; + MockRefreshImagesRequest mock_refresh_images_request; + expect_refresh_images(mock_refresh_images_request, image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0); + + MockListener mock_listener(this); + expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {}); + + MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx, + mock_listener); + C_SaferCond ctx; + mock_pool_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + ASSERT_TRUE(wait_for_update(1)); + + expect_timer_add_event(mock_threads); + ImageIds new_image_ids{ + {"global id 1", "remote id 1"}}; + expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false); + expect_refresh_images(mock_refresh_images_request, new_image_ids, 0); + expect_mirror_uuid_get(m_remote_io_ctx, "updated uuid", 0); + expect_listener_handle_update(mock_listener, "remote uuid", {}, image_ids); + expect_listener_handle_update(mock_listener, "updated uuid", new_image_ids, + {}); + + MirroringWatcher::get_instance().handle_rewatch_complete(0); + ASSERT_TRUE(wait_for_update(2)); + + expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_pool_watcher)); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/test/rbd_mirror/test_mock_fixture.cc b/src/test/rbd_mirror/test_mock_fixture.cc new file mode 100644 index 00000000..9e308a63 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_fixture.cc @@ -0,0 +1,64 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "include/rbd/librbd.hpp" +#include "test/librados_test_stub/LibradosTestStub.h" +#include "test/librados_test_stub/MockTestMemCluster.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace rbd { +namespace mirror { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::WithArg; + +TestMockFixture::TestClusterRef TestMockFixture::s_test_cluster; + +void TestMockFixture::SetUpTestCase() { + s_test_cluster = librados_test_stub::get_cluster(); + + // use a mock version of the in-memory rados client + librados_test_stub::set_cluster(boost::shared_ptr<librados::TestCluster>( + new ::testing::NiceMock<librados::MockTestMemCluster>())); + TestFixture::SetUpTestCase(); +} + +void TestMockFixture::TearDownTestCase() { + TestFixture::TearDownTestCase(); + librados_test_stub::set_cluster(s_test_cluster); +} + +void TestMockFixture::TearDown() { + // Mock rados client lives across tests -- reset it to initial state + librados::MockTestMemRadosClient *mock_rados_client = + get_mock_io_ctx(m_local_io_ctx).get_mock_rados_client(); + ASSERT_TRUE(mock_rados_client != nullptr); + + ::testing::Mock::VerifyAndClear(mock_rados_client); + mock_rados_client->default_to_dispatch(); + dynamic_cast<librados::MockTestMemCluster*>( + librados_test_stub::get_cluster().get())->default_to_dispatch(); + + TestFixture::TearDown(); +} + +void TestMockFixture::expect_test_features(librbd::MockImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, test_features(_, _)) + .WillRepeatedly(WithArg<0>(Invoke([&mock_image_ctx](uint64_t features) { + return (mock_image_ctx.features & features) != 0; + }))); +} + +librados::MockTestMemCluster& TestMockFixture::get_mock_cluster() { + librados::MockTestMemCluster* mock_cluster = dynamic_cast< + librados::MockTestMemCluster*>(librados_test_stub::get_cluster().get()); + ceph_assert(mock_cluster != nullptr); + return *mock_cluster; +} + +} // namespace mirror +} // namespace rbd + diff --git a/src/test/rbd_mirror/test_mock_fixture.h b/src/test/rbd_mirror/test_mock_fixture.h new file mode 100644 index 00000000..1f7dfb03 --- /dev/null +++ b/src/test/rbd_mirror/test_mock_fixture.h @@ -0,0 +1,71 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_TEST_RBD_MIRROR_TEST_MOCK_FIXTURE_H +#define CEPH_TEST_RBD_MIRROR_TEST_MOCK_FIXTURE_H + +#include "test/rbd_mirror/test_fixture.h" +#include "test/librados_test_stub/LibradosTestStub.h" +#include "common/WorkQueue.h" +#include <boost/shared_ptr.hpp> +#include <gmock/gmock.h> +#include "include/ceph_assert.h" + +namespace librados { +class TestRadosClient; +class MockTestMemCluster; +class MockTestMemIoCtxImpl; +class MockTestMemRadosClient; +} + +namespace librbd { +class MockImageCtx; +} + +ACTION_P(CopyInBufferlist, str) { + arg0->append(str); +} + +ACTION_P(CompleteContext, r) { + arg0->complete(r); +} + +ACTION_P2(CompleteContext, wq, r) { + ContextWQ *context_wq = reinterpret_cast<ContextWQ *>(wq); + context_wq->queue(arg0, r); +} + +ACTION_P(GetReference, ref_object) { + ref_object->get(); +} + +MATCHER_P(ContentsEqual, bl, "") { + // TODO fix const-correctness of bufferlist + return const_cast<bufferlist &>(arg).contents_equal( + const_cast<bufferlist &>(bl)); +} + +namespace rbd { +namespace mirror { + +class TestMockFixture : public TestFixture { +public: + typedef boost::shared_ptr<librados::TestCluster> TestClusterRef; + + static void SetUpTestCase(); + static void TearDownTestCase(); + + void TearDown() override; + + void expect_test_features(librbd::MockImageCtx &mock_image_ctx); + + librados::MockTestMemCluster& get_mock_cluster(); + +private: + static TestClusterRef s_test_cluster; +}; + +} // namespace mirror +} // namespace rbd + +#endif // CEPH_TEST_RBD_MIRROR_TEST_MOCK_FIXTURE_H |