From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:54:28 +0200 Subject: Adding upstream version 18.2.2. Signed-off-by: Daniel Baumann --- src/librbd/mirror/DemoteRequest.cc | 216 ++++++++++ src/librbd/mirror/DemoteRequest.h | 86 ++++ src/librbd/mirror/DisableRequest.cc | 479 +++++++++++++++++++++ src/librbd/mirror/DisableRequest.h | 143 ++++++ src/librbd/mirror/EnableRequest.cc | 329 ++++++++++++++ src/librbd/mirror/EnableRequest.h | 135 ++++++ src/librbd/mirror/GetInfoRequest.cc | 290 +++++++++++++ src/librbd/mirror/GetInfoRequest.h | 123 ++++++ src/librbd/mirror/GetStatusRequest.cc | 116 +++++ src/librbd/mirror/GetStatusRequest.h | 86 ++++ src/librbd/mirror/GetUuidRequest.cc | 86 ++++ src/librbd/mirror/GetUuidRequest.h | 69 +++ src/librbd/mirror/ImageRemoveRequest.cc | 98 +++++ src/librbd/mirror/ImageRemoveRequest.h | 77 ++++ src/librbd/mirror/ImageStateUpdateRequest.cc | 151 +++++++ src/librbd/mirror/ImageStateUpdateRequest.h | 92 ++++ src/librbd/mirror/PromoteRequest.cc | 115 +++++ src/librbd/mirror/PromoteRequest.h | 76 ++++ src/librbd/mirror/Types.h | 21 + .../mirror/snapshot/CreateNonPrimaryRequest.cc | 273 ++++++++++++ .../mirror/snapshot/CreateNonPrimaryRequest.h | 123 ++++++ src/librbd/mirror/snapshot/CreatePrimaryRequest.cc | 281 ++++++++++++ src/librbd/mirror/snapshot/CreatePrimaryRequest.h | 106 +++++ src/librbd/mirror/snapshot/DemoteRequest.cc | 110 +++++ src/librbd/mirror/snapshot/DemoteRequest.h | 76 ++++ src/librbd/mirror/snapshot/GetImageStateRequest.cc | 114 +++++ src/librbd/mirror/snapshot/GetImageStateRequest.h | 76 ++++ src/librbd/mirror/snapshot/ImageMeta.cc | 175 ++++++++ src/librbd/mirror/snapshot/ImageMeta.h | 78 ++++ src/librbd/mirror/snapshot/PromoteRequest.cc | 405 +++++++++++++++++ src/librbd/mirror/snapshot/PromoteRequest.h | 151 +++++++ .../mirror/snapshot/RemoveImageStateRequest.cc | 131 ++++++ .../mirror/snapshot/RemoveImageStateRequest.h | 75 ++++ src/librbd/mirror/snapshot/SetImageStateRequest.cc | 235 ++++++++++ src/librbd/mirror/snapshot/SetImageStateRequest.h | 96 +++++ src/librbd/mirror/snapshot/Types.cc | 109 +++++ src/librbd/mirror/snapshot/Types.h | 122 ++++++ src/librbd/mirror/snapshot/UnlinkPeerRequest.cc | 230 ++++++++++ src/librbd/mirror/snapshot/UnlinkPeerRequest.h | 98 +++++ src/librbd/mirror/snapshot/Utils.cc | 186 ++++++++ src/librbd/mirror/snapshot/Utils.h | 38 ++ .../mirror/snapshot/WriteImageStateRequest.cc | 120 ++++++ .../mirror/snapshot/WriteImageStateRequest.h | 73 ++++ 43 files changed, 6269 insertions(+) create mode 100644 src/librbd/mirror/DemoteRequest.cc create mode 100644 src/librbd/mirror/DemoteRequest.h create mode 100644 src/librbd/mirror/DisableRequest.cc create mode 100644 src/librbd/mirror/DisableRequest.h create mode 100644 src/librbd/mirror/EnableRequest.cc create mode 100644 src/librbd/mirror/EnableRequest.h create mode 100644 src/librbd/mirror/GetInfoRequest.cc create mode 100644 src/librbd/mirror/GetInfoRequest.h create mode 100644 src/librbd/mirror/GetStatusRequest.cc create mode 100644 src/librbd/mirror/GetStatusRequest.h create mode 100644 src/librbd/mirror/GetUuidRequest.cc create mode 100644 src/librbd/mirror/GetUuidRequest.h create mode 100644 src/librbd/mirror/ImageRemoveRequest.cc create mode 100644 src/librbd/mirror/ImageRemoveRequest.h create mode 100644 src/librbd/mirror/ImageStateUpdateRequest.cc create mode 100644 src/librbd/mirror/ImageStateUpdateRequest.h create mode 100644 src/librbd/mirror/PromoteRequest.cc create mode 100644 src/librbd/mirror/PromoteRequest.h create mode 100644 src/librbd/mirror/Types.h create mode 100644 src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc create mode 100644 src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h create mode 100644 src/librbd/mirror/snapshot/CreatePrimaryRequest.cc create mode 100644 src/librbd/mirror/snapshot/CreatePrimaryRequest.h create mode 100644 src/librbd/mirror/snapshot/DemoteRequest.cc create mode 100644 src/librbd/mirror/snapshot/DemoteRequest.h create mode 100644 src/librbd/mirror/snapshot/GetImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/GetImageStateRequest.h create mode 100644 src/librbd/mirror/snapshot/ImageMeta.cc create mode 100644 src/librbd/mirror/snapshot/ImageMeta.h create mode 100644 src/librbd/mirror/snapshot/PromoteRequest.cc create mode 100644 src/librbd/mirror/snapshot/PromoteRequest.h create mode 100644 src/librbd/mirror/snapshot/RemoveImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/RemoveImageStateRequest.h create mode 100644 src/librbd/mirror/snapshot/SetImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/SetImageStateRequest.h create mode 100644 src/librbd/mirror/snapshot/Types.cc create mode 100644 src/librbd/mirror/snapshot/Types.h create mode 100644 src/librbd/mirror/snapshot/UnlinkPeerRequest.cc create mode 100644 src/librbd/mirror/snapshot/UnlinkPeerRequest.h create mode 100644 src/librbd/mirror/snapshot/Utils.cc create mode 100644 src/librbd/mirror/snapshot/Utils.h create mode 100644 src/librbd/mirror/snapshot/WriteImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/WriteImageStateRequest.h (limited to 'src/librbd/mirror') diff --git a/src/librbd/mirror/DemoteRequest.cc b/src/librbd/mirror/DemoteRequest.cc new file mode 100644 index 000000000..350a76d83 --- /dev/null +++ b/src/librbd/mirror/DemoteRequest.cc @@ -0,0 +1,216 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/DemoteRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "librbd/mirror/snapshot/DemoteRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::DemoteRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; + +template +void DemoteRequest::send() { + get_info(); +} + +template +void DemoteRequest::get_info() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_get_info>(this); + auto req = GetInfoRequest::create(m_image_ctx, &m_mirror_image, + &m_promotion_state, + &m_primary_mirror_uuid, ctx); + req->send(); +} + +template +void DemoteRequest::handle_get_info(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else if (m_mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(cct) << "mirroring is not currently enabled" << dendl; + finish(-EINVAL); + return; + } else if (m_promotion_state != PROMOTION_STATE_PRIMARY) { + lderr(cct) << "image is not primary" << dendl; + finish(-EINVAL); + return; + } + + acquire_lock(); +} + +template +void DemoteRequest::acquire_lock() { + CephContext *cct = m_image_ctx.cct; + + m_image_ctx.owner_lock.lock_shared(); + if (m_image_ctx.exclusive_lock == nullptr) { + m_image_ctx.owner_lock.unlock_shared(); + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + lderr(cct) << "exclusive lock is not active" << dendl; + finish(-EINVAL); + } else { + demote(); + } + return; + } + + // avoid accepting new requests from peers while we demote + // the image + m_image_ctx.exclusive_lock->block_requests(0); + m_blocked_requests = true; + + if (m_image_ctx.exclusive_lock->is_lock_owner()) { + m_image_ctx.owner_lock.unlock_shared(); + demote(); + return; + } + + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, + &DemoteRequest::handle_acquire_lock>(this, m_image_ctx.exclusive_lock); + m_image_ctx.exclusive_lock->acquire_lock(ctx); + m_image_ctx.owner_lock.unlock_shared(); +} + +template +void DemoteRequest::handle_acquire_lock(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to lock image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_image_ctx.owner_lock.lock_shared(); + if (m_image_ctx.exclusive_lock != nullptr && + !m_image_ctx.exclusive_lock->is_lock_owner()) { + r = m_image_ctx.exclusive_lock->get_unlocked_op_error(); + m_image_ctx.owner_lock.unlock_shared(); + lderr(cct) << "failed to acquire exclusive lock" << dendl; + finish(r); + return; + } + m_image_ctx.owner_lock.unlock_shared(); + + demote(); +} + +template +void DemoteRequest::demote() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_demote>(this); + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + Journal::demote(&m_image_ctx, ctx); + } else if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto req = mirror::snapshot::DemoteRequest::create( + &m_image_ctx, m_mirror_image.global_image_id, ctx); + req->send(); + } else { + lderr(cct) << "unknown image mirror mode: " << m_mirror_image.mode << dendl; + m_ret_val = -EOPNOTSUPP; + release_lock(); + } +} + +template +void DemoteRequest::handle_demote(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + m_ret_val = r; + lderr(cct) << "failed to demote image: " << cpp_strerror(r) << dendl; + } + + release_lock(); +} + +template +void DemoteRequest::release_lock() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + m_image_ctx.owner_lock.lock_shared(); + if (m_image_ctx.exclusive_lock == nullptr) { + m_image_ctx.owner_lock.unlock_shared(); + finish(0); + return; + } + + auto ctx = create_context_callback< + DemoteRequest, + &DemoteRequest::handle_release_lock>(this, m_image_ctx.exclusive_lock); + m_image_ctx.exclusive_lock->release_lock(ctx); + m_image_ctx.owner_lock.unlock_shared(); +} + +template +void DemoteRequest::handle_release_lock(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to release exclusive lock: " << cpp_strerror(r) + << dendl; + } + + finish(r); +} + +template +void DemoteRequest::finish(int r) { + if (m_ret_val < 0) { + r = m_ret_val; + } + + { + std::shared_lock owner_locker{m_image_ctx.owner_lock}; + if (m_blocked_requests && m_image_ctx.exclusive_lock != nullptr) { + m_image_ctx.exclusive_lock->unblock_requests(); + } + } + + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::DemoteRequest; diff --git a/src/librbd/mirror/DemoteRequest.h b/src/librbd/mirror/DemoteRequest.h new file mode 100644 index 000000000..ab9239068 --- /dev/null +++ b/src/librbd/mirror/DemoteRequest.h @@ -0,0 +1,86 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_DEMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_DEMOTE_REQUEST_H + +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class DemoteRequest { +public: + static DemoteRequest *create(ImageCtxT &image_ctx, Context *on_finish) { + return new DemoteRequest(image_ctx, on_finish); + } + + DemoteRequest(ImageCtxT &image_ctx, Context *on_finish) + : m_image_ctx(image_ctx), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_INFO + * | + * v + * ACQUIRE_LOCK * * * * + * | * + * v * + * DEMOTE * + * | * + * v * + * RELEASE_LOCK * + * | * + * v * + * < * * * * * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + Context *m_on_finish; + + int m_ret_val = 0; + bool m_blocked_requests = false; + + cls::rbd::MirrorImage m_mirror_image; + PromotionState m_promotion_state = PROMOTION_STATE_PRIMARY; + std::string m_primary_mirror_uuid; + + void get_info(); + void handle_get_info(int r); + + void acquire_lock(); + void handle_acquire_lock(int r); + + void demote(); + void handle_demote(int r); + + void release_lock(); + void handle_release_lock(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::DemoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_DEMOTE_REQUEST_H diff --git a/src/librbd/mirror/DisableRequest.cc b/src/librbd/mirror/DisableRequest.cc new file mode 100644 index 000000000..09378ce58 --- /dev/null +++ b/src/librbd/mirror/DisableRequest.cc @@ -0,0 +1,479 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/DisableRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/journal/cls_journal_client.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/journal/PromoteRequest.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "librbd/mirror/ImageRemoveRequest.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" +#include "librbd/mirror/snapshot/PromoteRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::DisableRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using util::create_rados_callback; + +template +DisableRequest::DisableRequest(I *image_ctx, bool force, bool remove, + Context *on_finish) + : m_image_ctx(image_ctx), m_force(force), m_remove(remove), + m_on_finish(on_finish) { +} + +template +void DisableRequest::send() { + send_get_mirror_info(); +} + +template +void DisableRequest::send_get_mirror_info() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + + using klass = DisableRequest; + Context *ctx = util::create_context_callback< + klass, &klass::handle_get_mirror_info>(this); + + auto req = GetInfoRequest::create(*m_image_ctx, &m_mirror_image, + &m_promotion_state, + &m_primary_mirror_uuid, ctx); + req->send(); +} + +template +Context *DisableRequest::handle_get_mirror_info(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + if (*result < 0) { + if (*result == -ENOENT) { + ldout(cct, 20) << "mirroring is not enabled for this image" << dendl; + *result = 0; + } else { + lderr(cct) << "failed to get mirroring info: " << cpp_strerror(*result) + << dendl; + } + return m_on_finish; + } + + m_is_primary = (m_promotion_state == PROMOTION_STATE_PRIMARY || + m_promotion_state == PROMOTION_STATE_UNKNOWN); + + if (!m_is_primary && !m_force) { + lderr(cct) << "mirrored image is not primary, " + << "add force option to disable mirroring" << dendl; + *result = -EINVAL; + return m_on_finish; + } + + send_image_state_update(); + return nullptr; +} + +template +void DisableRequest::send_image_state_update() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + auto ctx = util::create_context_callback< + DisableRequest, + &DisableRequest::handle_image_state_update>(this); + auto req = ImageStateUpdateRequest::create( + m_image_ctx->md_ctx, m_image_ctx->id, + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, m_mirror_image, ctx); + req->send(); +} + +template +Context *DisableRequest::handle_image_state_update(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to disable mirroring: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + send_promote_image(); + return nullptr; +} + +template +void DisableRequest::send_promote_image() { + if (m_is_primary) { + clean_mirror_state(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + auto ctx = util::create_context_callback< + DisableRequest, &DisableRequest::handle_promote_image>(this); + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + // Not primary -- shouldn't have the journal open + ceph_assert(m_image_ctx->journal == nullptr); + + auto req = journal::PromoteRequest::create(m_image_ctx, true, ctx); + req->send(); + } else if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto req = mirror::snapshot::PromoteRequest::create( + m_image_ctx, m_mirror_image.global_image_id, ctx); + req->send(); + } else { + lderr(cct) << "unknown image mirror mode: " << m_mirror_image.mode << dendl; + ctx->complete(-EOPNOTSUPP); + } +} + +template +Context *DisableRequest::handle_promote_image(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to promote image: " << cpp_strerror(*result) << dendl; + return m_on_finish; + } + + send_refresh_image(); + return nullptr; +} + +template +void DisableRequest::send_refresh_image() { + if (!m_image_ctx->state->is_refresh_required()) { + clean_mirror_state(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + auto ctx = util::create_context_callback< + DisableRequest, + &DisableRequest::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template +Context *DisableRequest::handle_refresh_image(int* result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(*result) << dendl; + return m_on_finish; + } + + clean_mirror_state(); + return nullptr; +} + +template +void DisableRequest::clean_mirror_state() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + remove_mirror_snapshots(); + } else { + send_get_clients(); + } +} + +template +void DisableRequest::send_get_clients() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + using klass = DisableRequest; + Context *ctx = util::create_context_callback< + klass, &klass::handle_get_clients>(this); + + std::string header_oid = ::journal::Journaler::header_oid(m_image_ctx->id); + m_clients.clear(); + cls::journal::client::client_list(m_image_ctx->md_ctx, header_oid, &m_clients, + ctx); +} + +template +Context *DisableRequest::handle_get_clients(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + std::unique_lock locker{m_lock}; + ceph_assert(m_current_ops.empty()); + + if (*result < 0) { + lderr(cct) << "failed to get registered clients: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + for (auto client : m_clients) { + journal::ClientData client_data; + auto bl_it = client.data.cbegin(); + try { + using ceph::decode; + decode(client_data, bl_it); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode client data" << dendl; + m_error_result = -EBADMSG; + continue; + } + + journal::ClientMetaType type = client_data.get_client_meta_type(); + if (type != journal::ClientMetaType::MIRROR_PEER_CLIENT_META_TYPE) { + continue; + } + + if (m_current_ops.find(client.id) != m_current_ops.end()) { + // Should not happen. + lderr(cct) << "clients with the same id " + << client.id << dendl; + continue; + } + + m_current_ops[client.id] = 0; + m_ret[client.id] = 0; + + journal::MirrorPeerClientMeta client_meta = + boost::get(client_data.client_meta); + + for (const auto& sync : client_meta.sync_points) { + send_remove_snap(client.id, sync.snap_namespace, sync.snap_name); + } + + if (m_current_ops[client.id] == 0) { + // no snaps to remove + send_unregister_client(client.id); + } + } + + if (m_current_ops.empty()) { + if (m_error_result < 0) { + *result = m_error_result; + return m_on_finish; + } else if (!m_remove) { + return m_on_finish; + } + locker.unlock(); + + // no mirror clients to unregister + send_remove_mirror_image(); + } + + return nullptr; +} + +template +void DisableRequest::remove_mirror_snapshots() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // remove snapshot-based mirroring snapshots + bool removing_snapshots = false; + { + std::lock_guard locker{m_lock}; + std::shared_lock image_locker{m_image_ctx->image_lock}; + + for (auto &it : m_image_ctx->snap_info) { + auto &snap_info = it.second; + auto type = cls::rbd::get_snap_namespace_type( + snap_info.snap_namespace); + if (type == cls::rbd::SNAPSHOT_NAMESPACE_TYPE_MIRROR) { + send_remove_snap("", snap_info.snap_namespace, snap_info.name); + removing_snapshots = true; + } + } + } + + if (!removing_snapshots) { + send_remove_mirror_image(); + } +} + +template +void DisableRequest::send_remove_snap( + const std::string &client_id, + const cls::rbd::SnapshotNamespace &snap_namespace, + const std::string &snap_name) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "client_id=" << client_id + << ", snap_name=" << snap_name << dendl; + + ceph_assert(ceph_mutex_is_locked(m_lock)); + + m_current_ops[client_id]++; + + Context *ctx = create_context_callback( + &DisableRequest::handle_remove_snap, client_id); + + ctx = new LambdaContext([this, snap_namespace, snap_name, ctx](int r) { + m_image_ctx->operations->snap_remove(snap_namespace, + snap_name.c_str(), + ctx); + }); + + m_image_ctx->op_work_queue->queue(ctx, 0); +} + +template +Context *DisableRequest::handle_remove_snap(int *result, + const std::string &client_id) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + std::unique_lock locker{m_lock}; + + ceph_assert(m_current_ops[client_id] > 0); + m_current_ops[client_id]--; + + if (*result < 0 && *result != -ENOENT) { + lderr(cct) << "failed to remove mirroring snapshot: " + << cpp_strerror(*result) << dendl; + m_ret[client_id] = *result; + } + + if (m_current_ops[client_id] == 0) { + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + ceph_assert(client_id.empty()); + m_current_ops.erase(client_id); + if (m_ret[client_id] < 0) { + return m_on_finish; + } + locker.unlock(); + + send_remove_mirror_image(); + return nullptr; + } + + send_unregister_client(client_id); + } + + return nullptr; +} + +template +void DisableRequest::send_unregister_client( + const std::string &client_id) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + ceph_assert(ceph_mutex_is_locked(m_lock)); + ceph_assert(m_current_ops[client_id] == 0); + + Context *ctx = create_context_callback( + &DisableRequest::handle_unregister_client, client_id); + + if (m_ret[client_id] < 0) { + m_image_ctx->op_work_queue->queue(ctx, m_ret[client_id]); + return; + } + + librados::ObjectWriteOperation op; + cls::journal::client::client_unregister(&op, client_id); + std::string header_oid = ::journal::Journaler::header_oid(m_image_ctx->id); + librados::AioCompletion *comp = create_rados_callback(ctx); + + int r = m_image_ctx->md_ctx.aio_operate(header_oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +Context *DisableRequest::handle_unregister_client( + int *result, const std::string &client_id) { + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + std::unique_lock locker{m_lock}; + ceph_assert(m_current_ops[client_id] == 0); + m_current_ops.erase(client_id); + + if (*result < 0 && *result != -ENOENT) { + lderr(cct) << "failed to unregister remote journal client: " + << cpp_strerror(*result) << dendl; + m_error_result = *result; + } + + if (!m_current_ops.empty()) { + return nullptr; + } + + if (m_error_result < 0) { + *result = m_error_result; + return m_on_finish; + } + locker.unlock(); + + send_get_clients(); + return nullptr; +} + +template +void DisableRequest::send_remove_mirror_image() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + auto ctx = util::create_context_callback< + DisableRequest, + &DisableRequest::handle_remove_mirror_image>(this); + auto req = ImageRemoveRequest::create( + m_image_ctx->md_ctx, m_mirror_image.global_image_id, m_image_ctx->id, + ctx); + req->send(); +} + +template +Context *DisableRequest::handle_remove_mirror_image(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << *result << dendl; + + if (*result < 0) { + lderr(cct) << "failed to remove mirror image: " << cpp_strerror(*result) + << dendl; + return m_on_finish; + } + + ldout(cct, 20) << "removed image state from rbd_mirroring object" << dendl; + return m_on_finish; +} + +template +Context *DisableRequest::create_context_callback( + Context*(DisableRequest::*handle)(int*, const std::string &client_id), + const std::string &client_id) { + + return new LambdaContext([this, handle, client_id](int r) { + Context *on_finish = (this->*handle)(&r, client_id); + if (on_finish != nullptr) { + on_finish->complete(r); + delete this; + } + }); +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::DisableRequest; diff --git a/src/librbd/mirror/DisableRequest.h b/src/librbd/mirror/DisableRequest.h new file mode 100644 index 000000000..f45d1a14c --- /dev/null +++ b/src/librbd/mirror/DisableRequest.h @@ -0,0 +1,143 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_DISABLE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_DISABLE_REQUEST_H + +#include "include/buffer.h" +#include "common/ceph_mutex.h" +#include "cls/journal/cls_journal_types.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" + +#include +#include + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace mirror { + +template +class DisableRequest { +public: + static DisableRequest *create(ImageCtxT *image_ctx, bool force, + bool remove, Context *on_finish) { + return new DisableRequest(image_ctx, force, remove, on_finish); + } + + DisableRequest(ImageCtxT *image_ctx, bool force, bool remove, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_MIRROR_INFO * * * * * * * * * * * * * * * * * * * * * * * + * | * + * v * + * IMAGE_STATE_UPDATE * * * * * * * * * * * * * * * * * * * * * * + * | * + * v * + * PROMOTE_IMAGE (skip if primary) * + * | * + * v * + * REFRESH_IMAGE (skip if necessary) * + * | * + * v * + * GET_CLIENTS <----------------------------------------\ * * * * + * | | (unregister clients) | * (on error) + * | |/----------------------------\ | * + * | | | | * + * | | /-----------\ (repeat | (repeat | (repeat + * | | | | as needed) | as needed) | as needed) + * | v v | | | * + * | REMOVE_SYNC_SNAP --/ * * * * * * | * * * * * * | * * * * + * | | | | * + * | v | | * + * | UNREGISTER_CLIENT ---------------/-------------/ * * * * + * | * + * | (no more clients * + * | to unregister) * + * v * + * REMOVE_MIRROR_IMAGE * * * * * * * * * * * * * * * * * * * * * + * | (skip if no remove) * + * v * + * < * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + bool m_force; + bool m_remove; + Context *m_on_finish; + + bool m_is_primary = false; + cls::rbd::MirrorImage m_mirror_image; + PromotionState m_promotion_state = PROMOTION_STATE_NON_PRIMARY; + std::string m_primary_mirror_uuid; + std::set m_clients; + std::map m_ret; + std::map m_current_ops; + int m_error_result = 0; + mutable ceph::mutex m_lock = + ceph::make_mutex("mirror::DisableRequest::m_lock"); + + void send_get_mirror_info(); + Context *handle_get_mirror_info(int *result); + + void send_image_state_update(); + Context *handle_image_state_update(int *result); + + void send_notify_mirroring_watcher(); + Context *handle_notify_mirroring_watcher(int *result); + + void send_promote_image(); + Context *handle_promote_image(int *result); + + void send_refresh_image(); + Context* handle_refresh_image(int* result); + + void clean_mirror_state(); + + void send_get_clients(); + Context *handle_get_clients(int *result); + + void remove_mirror_snapshots(); + + void send_remove_snap(const std::string &client_id, + const cls::rbd::SnapshotNamespace &snap_namespace, + const std::string &snap_name); + Context *handle_remove_snap(int *result, const std::string &client_id); + + void send_unregister_client(const std::string &client_id); + Context *handle_unregister_client(int *result, const std::string &client_id); + + void send_remove_mirror_image(); + Context *handle_remove_mirror_image(int *result); + + void send_notify_mirroring_watcher_removed(); + Context *handle_notify_mirroring_watcher_removed(int *result); + + Context *create_context_callback( + Context*(DisableRequest::*handle)( + int*, const std::string &client_id), + const std::string &client_id); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::DisableRequest; + +#endif // CEPH_LIBRBD_MIRROR_DISABLE_REQUEST_H diff --git a/src/librbd/mirror/EnableRequest.cc b/src/librbd/mirror/EnableRequest.cc new file mode 100644 index 000000000..fd74a25ba --- /dev/null +++ b/src/librbd/mirror/EnableRequest.cc @@ -0,0 +1,329 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/EnableRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::EnableRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using util::create_context_callback; +using util::create_rados_callback; + +template +EnableRequest::EnableRequest(librados::IoCtx &io_ctx, + const std::string &image_id, + I* image_ctx, + cls::rbd::MirrorImageMode mode, + const std::string &non_primary_global_image_id, + bool image_clean, + asio::ContextWQ *op_work_queue, + Context *on_finish) + : m_io_ctx(io_ctx), m_image_id(image_id), m_image_ctx(image_ctx), + m_mode(mode), m_non_primary_global_image_id(non_primary_global_image_id), + m_image_clean(image_clean), m_op_work_queue(op_work_queue), + m_on_finish(on_finish), + m_cct(reinterpret_cast(io_ctx.cct())) { +} + +template +void EnableRequest::send() { + get_mirror_image(); +} + +template +void EnableRequest::get_mirror_image() { + ldout(m_cct, 10) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_id); + + using klass = EnableRequest; + librados::AioCompletion *comp = + create_rados_callback(this); + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void EnableRequest::handle_get_mirror_image(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_image_get_finish(&iter, &m_mirror_image); + } + + if (r == 0 && m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_CREATING && + !m_non_primary_global_image_id.empty()) { + // special case where rbd-mirror injects a disabled record to record the + // local image id prior to creating ther image + ldout(m_cct, 10) << "enabling mirroring on in-progress image replication" + << dendl; + } else if (r == 0) { + if (m_mirror_image.mode != m_mode) { + lderr(m_cct) << "invalid current image mirror mode" << dendl; + r = -EINVAL; + } else if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + ldout(m_cct, 10) << "mirroring is already enabled" << dendl; + } else { + lderr(m_cct) << "currently disabling" << dendl; + r = -EINVAL; + } + finish(r); + return; + } else if (r != -ENOENT) { + lderr(m_cct) << "failed to retrieve mirror image: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + r = 0; + m_mirror_image.mode = m_mode; + if (m_non_primary_global_image_id.empty()) { + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_mirror_image.global_image_id = uuid_gen.to_string(); + } else { + m_mirror_image.global_image_id = m_non_primary_global_image_id; + } + + get_tag_owner(); +} + +template +void EnableRequest::get_tag_owner() { + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + open_image(); + return; + } else if (!m_non_primary_global_image_id.empty()) { + image_state_update(); + return; + } + + ldout(m_cct, 10) << dendl; + + using klass = EnableRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_get_tag_owner>(this); + librbd::Journal<>::is_tag_owner(m_io_ctx, m_image_id, &m_is_primary, + m_op_work_queue, ctx); +} + +template +void EnableRequest::handle_get_tag_owner(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to check tag ownership: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (!m_is_primary) { + lderr(m_cct) << "last journal tag not owned by local cluster" << dendl; + finish(-EINVAL); + return; + } + + image_state_update(); +} + +template +void EnableRequest::open_image() { + if (!m_non_primary_global_image_id.empty()) { + // special case for rbd-mirror creating a non-primary image + enable_non_primary_feature(); + return; + } else if (m_image_ctx != nullptr) { + create_primary_snapshot(); + return; + } + + ldout(m_cct, 10) << dendl; + + m_close_image = true; + m_image_ctx = I::create("", m_image_id, CEPH_NOSNAP, m_io_ctx, false); + + auto ctx = create_context_callback< + EnableRequest, &EnableRequest::handle_open_image>(this); + m_image_ctx->state->open(OPEN_FLAG_SKIP_OPEN_PARENT | + OPEN_FLAG_IGNORE_MIGRATING, ctx); +} + +template +void EnableRequest::handle_open_image(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to open image: " << cpp_strerror(r) << dendl; + m_image_ctx = nullptr; + finish(r); + return; + } + + create_primary_snapshot(); +} + +template +void EnableRequest::create_primary_snapshot() { + ldout(m_cct, 10) << dendl; + + ceph_assert(m_image_ctx != nullptr); + uint64_t snap_create_flags; + int r = util::snap_create_flags_api_to_internal( + m_cct, util::get_default_snap_create_flags(m_image_ctx), + &snap_create_flags); + ceph_assert(r == 0); + auto ctx = create_context_callback< + EnableRequest, + &EnableRequest::handle_create_primary_snapshot>(this); + auto req = snapshot::CreatePrimaryRequest::create( + m_image_ctx, m_mirror_image.global_image_id, + (m_image_clean ? 0 : CEPH_NOSNAP), snap_create_flags, + snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS, &m_snap_id, ctx); + req->send(); +} + +template +void EnableRequest::handle_create_primary_snapshot(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to create initial primary snapshot: " + << cpp_strerror(r) << dendl; + m_ret_val = r; + } + + close_image(); +} + +template +void EnableRequest::close_image() { + if (!m_close_image) { + if (m_ret_val < 0) { + finish(m_ret_val); + } else { + image_state_update(); + } + return; + } + + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + EnableRequest, &EnableRequest::handle_close_image>(this); + m_image_ctx->state->close(ctx); +} + +template +void EnableRequest::handle_close_image(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_image_ctx = nullptr; + + if (r < 0) { + lderr(m_cct) << "failed to close image: " << cpp_strerror(r) << dendl; + if (m_ret_val == 0) { + m_ret_val = r; + } + } + + if (m_ret_val < 0) { + finish(m_ret_val); + return; + } + + image_state_update(); +} + + +template +void EnableRequest::enable_non_primary_feature() { + if (m_mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + image_state_update(); + return; + } + + ldout(m_cct, 10) << dendl; + + // ensure image is flagged with non-primary feature so that + // standard RBD clients cannot write to it. + librados::ObjectWriteOperation op; + cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY, + RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + EnableRequest, + &EnableRequest::handle_enable_non_primary_feature>(this); + int r = m_io_ctx.aio_operate(util::header_name(m_image_id), aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void EnableRequest::handle_enable_non_primary_feature(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to enable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void EnableRequest::image_state_update() { + ldout(m_cct, 10) << dendl; + + auto ctx = create_context_callback< + EnableRequest, &EnableRequest::handle_image_state_update>(this); + auto req = ImageStateUpdateRequest::create( + m_io_ctx, m_image_id, cls::rbd::MIRROR_IMAGE_STATE_ENABLED, + m_mirror_image, ctx); + req->send(); +} + +template +void EnableRequest::handle_image_state_update(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to enable mirroring: " << cpp_strerror(r) + << dendl; + } + + finish(r); +} + +template +void EnableRequest::finish(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::EnableRequest; diff --git a/src/librbd/mirror/EnableRequest.h b/src/librbd/mirror/EnableRequest.h new file mode 100644 index 000000000..391028e6e --- /dev/null +++ b/src/librbd/mirror/EnableRequest.h @@ -0,0 +1,135 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_ENABLE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_ENABLE_REQUEST_H + +#include "include/buffer_fwd.h" +#include "include/rados/librados_fwd.hpp" +#include "include/rbd/librbd.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/mirror/Types.h" +#include +#include + +class Context; + +namespace librbd { + +namespace asio { struct ContextWQ; } + +namespace mirror { + +template +class EnableRequest { +public: + static EnableRequest *create(ImageCtxT *image_ctx, + cls::rbd::MirrorImageMode mode, + const std::string &non_primary_global_image_id, + bool image_clean, Context *on_finish) { + return new EnableRequest(image_ctx->md_ctx, image_ctx->id, image_ctx, mode, + non_primary_global_image_id, image_clean, + image_ctx->op_work_queue, on_finish); + } + static EnableRequest *create(librados::IoCtx &io_ctx, + const std::string &image_id, + cls::rbd::MirrorImageMode mode, + const std::string &non_primary_global_image_id, + bool image_clean, asio::ContextWQ *op_work_queue, + Context *on_finish) { + return new EnableRequest(io_ctx, image_id, nullptr, mode, + non_primary_global_image_id, image_clean, + op_work_queue, on_finish); + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_MIRROR_IMAGE * * * * * * * + * | * (on error) + * v (skip if not needed) * + * GET_TAG_OWNER * * * * * * * * + * | * + * v (skip if not needed) * + * OPEN_IMAGE * + * | * + * v (skip if not needed) * + * CREATE_PRIMARY_SNAPSHOT * * * + * | * + * v (skip of not opened) * + * CLOSE_IMAGE * + * | * + * v (skip if not needed) * + * ENABLE_NON_PRIMARY_FEATURE * + * | * + * v (skip if not needed) * + * IMAGE_STATE_UPDATE * * * * * * + * | * + * v * + * < * * * * * * * * * + * + * @endverbatim + */ + + EnableRequest(librados::IoCtx &io_ctx, const std::string &image_id, + ImageCtxT* image_ctx, cls::rbd::MirrorImageMode mode, + const std::string &non_primary_global_image_id, + bool image_clean, asio::ContextWQ *op_work_queue, + Context *on_finish); + + librados::IoCtx &m_io_ctx; + std::string m_image_id; + ImageCtxT* m_image_ctx; + cls::rbd::MirrorImageMode m_mode; + std::string m_non_primary_global_image_id; + bool m_image_clean; + asio::ContextWQ *m_op_work_queue; + Context *m_on_finish; + + CephContext *m_cct = nullptr; + bufferlist m_out_bl; + cls::rbd::MirrorImage m_mirror_image; + + int m_ret_val = 0; + bool m_close_image = false; + + bool m_is_primary = false; + uint64_t m_snap_id = CEPH_NOSNAP; + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void get_tag_owner(); + void handle_get_tag_owner(int r); + + void open_image(); + void handle_open_image(int r); + + void create_primary_snapshot(); + void handle_create_primary_snapshot(int r); + + void close_image(); + void handle_close_image(int r); + + void enable_non_primary_feature(); + void handle_enable_non_primary_feature(int r); + + void image_state_update(); + void handle_image_state_update(int r); + + void finish(int r); +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::EnableRequest; + +#endif // CEPH_LIBRBD_MIRROR_ENABLE_REQUEST_H diff --git a/src/librbd/mirror/GetInfoRequest.cc b/src/librbd/mirror/GetInfoRequest.cc new file mode 100644 index 000000000..2db8aaa84 --- /dev/null +++ b/src/librbd/mirror/GetInfoRequest.cc @@ -0,0 +1,290 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/GetInfoRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::GetInfoRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +GetInfoRequest::GetInfoRequest(librados::IoCtx& io_ctx, + asio::ContextWQ *op_work_queue, + const std::string &image_id, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) + : m_io_ctx(io_ctx), m_op_work_queue(op_work_queue), m_image_id(image_id), + m_mirror_image(mirror_image), m_promotion_state(promotion_state), + m_primary_mirror_uuid(primary_mirror_uuid), m_on_finish(on_finish), + m_cct(reinterpret_cast(io_ctx.cct())) { +} + +template +GetInfoRequest::GetInfoRequest(I &image_ctx, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) + : m_image_ctx(&image_ctx), m_io_ctx(image_ctx.md_ctx), + m_op_work_queue(image_ctx.op_work_queue), m_image_id(image_ctx.id), + m_mirror_image(mirror_image), m_promotion_state(promotion_state), + m_primary_mirror_uuid(primary_mirror_uuid), m_on_finish(on_finish), + m_cct(image_ctx.cct) { +} + +template +void GetInfoRequest::send() { + get_mirror_image(); +} + +template +void GetInfoRequest::get_mirror_image() { + ldout(m_cct, 20) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_id); + + librados::AioCompletion *comp = create_rados_callback< + GetInfoRequest, &GetInfoRequest::handle_get_mirror_image>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GetInfoRequest::handle_get_mirror_image(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + m_mirror_image->state = cls::rbd::MIRROR_IMAGE_STATE_DISABLED; + *m_promotion_state = PROMOTION_STATE_NON_PRIMARY; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_image_get_finish(&iter, m_mirror_image); + } + + if (r == -ENOENT) { + ldout(m_cct, 20) << "mirroring is disabled" << dendl; + finish(r); + return; + } else if (r < 0) { + lderr(m_cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_mirror_image->mode == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + get_journal_tag_owner(); + } else if (m_mirror_image->mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + get_snapcontext(); + } else { + ldout(m_cct, 20) << "unknown mirror image mode: " << m_mirror_image->mode + << dendl; + finish(-EOPNOTSUPP); + } +} + +template +void GetInfoRequest::get_journal_tag_owner() { + ldout(m_cct, 20) << dendl; + + auto ctx = create_context_callback< + GetInfoRequest, &GetInfoRequest::handle_get_journal_tag_owner>(this); + Journal::get_tag_owner(m_io_ctx, m_image_id, &m_mirror_uuid, + m_op_work_queue, ctx); +} + +template +void GetInfoRequest::handle_get_journal_tag_owner(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to determine tag ownership: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { + *m_promotion_state = PROMOTION_STATE_PRIMARY; + *m_primary_mirror_uuid = ""; + } else if (m_mirror_uuid == Journal<>::ORPHAN_MIRROR_UUID) { + *m_promotion_state = PROMOTION_STATE_ORPHAN; + *m_primary_mirror_uuid = ""; + } else { + *m_primary_mirror_uuid = m_mirror_uuid; + } + + finish(0); +} + +template +void GetInfoRequest::get_snapcontext() { + if (m_image_ctx != nullptr) { + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + calc_promotion_state(m_image_ctx->snap_info); + } + finish(0); + return; + } + + ldout(m_cct, 20) << dendl; + + librados::ObjectReadOperation op; + cls_client::get_snapcontext_start(&op); + + librados::AioCompletion *comp = create_rados_callback< + GetInfoRequest, &GetInfoRequest::handle_get_snapcontext>(this); + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(util::header_name(m_image_id), comp, &op, + &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GetInfoRequest::handle_get_snapcontext(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + if (r >= 0) { + auto it = m_out_bl.cbegin(); + r = cls_client::get_snapcontext_finish(&it, &m_snapc); + } + + if (r == -ENOENT && + m_mirror_image->state == cls::rbd::MIRROR_IMAGE_STATE_CREATING) { + // image doesn't exist but we have a mirror image record for it + ldout(m_cct, 10) << "image does not exist for mirror image id " + << m_image_id << dendl; + *m_promotion_state = PROMOTION_STATE_UNKNOWN; + *m_primary_mirror_uuid = ""; + finish(0); + return; + } else if (r < 0) { + lderr(m_cct) << "failed to get snapcontext: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + get_snapshots(); +} + + +template +void GetInfoRequest::get_snapshots() { + ldout(m_cct, 20) << dendl; + + if (m_snapc.snaps.empty()) { + handle_get_snapshots(0); + return; + } + + librados::ObjectReadOperation op; + for (auto snap_id : m_snapc.snaps) { + cls_client::snapshot_get_start(&op, snap_id); + } + + librados::AioCompletion *comp = create_rados_callback< + GetInfoRequest, &GetInfoRequest::handle_get_snapshots>(this); + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(util::header_name(m_image_id), comp, &op, + &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GetInfoRequest::handle_get_snapshots(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + std::map snap_info; + + auto it = m_out_bl.cbegin(); + for (auto snap_id : m_snapc.snaps) { + cls::rbd::SnapshotInfo snap; + if (r >= 0) { + r = cls_client::snapshot_get_finish(&it, &snap); + } + snap_info.emplace( + snap_id, SnapInfo(snap.name, snap.snapshot_namespace, 0, {}, 0, 0, {})); + } + + if (r == -ENOENT) { + // restart + get_snapcontext(); + return; + } + + if (r < 0) { + lderr(m_cct) << "failed to get snapshots: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + calc_promotion_state(snap_info); + finish(0); +} + +template +void GetInfoRequest::finish(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +template +void GetInfoRequest::calc_promotion_state( + const std::map &snap_info) { + *m_promotion_state = PROMOTION_STATE_UNKNOWN; + *m_primary_mirror_uuid = ""; + + for (auto it = snap_info.rbegin(); it != snap_info.rend(); it++) { + auto mirror_ns = std::get_if( + &it->second.snap_namespace); + + if (mirror_ns != nullptr) { + switch (mirror_ns->state) { + case cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY: + *m_promotion_state = PROMOTION_STATE_PRIMARY; + break; + case cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY: + *m_promotion_state = PROMOTION_STATE_NON_PRIMARY; + *m_primary_mirror_uuid = mirror_ns->primary_mirror_uuid; + break; + case cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED: + case cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED: + *m_promotion_state = PROMOTION_STATE_ORPHAN; + break; + } + break; + } + } + + ldout(m_cct, 10) << "promotion_state=" << *m_promotion_state << ", " + << "primary_mirror_uuid=" << *m_primary_mirror_uuid << dendl; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::GetInfoRequest; diff --git a/src/librbd/mirror/GetInfoRequest.h b/src/librbd/mirror/GetInfoRequest.h new file mode 100644 index 000000000..dcc6da7da --- /dev/null +++ b/src/librbd/mirror/GetInfoRequest.h @@ -0,0 +1,123 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_GET_INFO_REQUEST_H +#define CEPH_LIBRBD_MIRROR_GET_INFO_REQUEST_H + +#include "common/snap_types.h" +#include "include/buffer.h" +#include "include/common_fwd.h" +#include "include/rados/librados.hpp" +#include "librbd/Types.h" +#include "librbd/mirror/Types.h" +#include + +struct Context; + +namespace cls { namespace rbd { struct MirrorImage; } } + +namespace librbd { + +struct ImageCtx; +namespace asio { struct ContextWQ; } + +namespace mirror { + +template +class GetInfoRequest { +public: + static GetInfoRequest *create(librados::IoCtx &io_ctx, + asio::ContextWQ *op_work_queue, + const std::string &image_id, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) { + return new GetInfoRequest(io_ctx, op_work_queue, image_id, mirror_image, + promotion_state, primary_mirror_uuid, on_finish); + } + static GetInfoRequest *create(ImageCtxT &image_ctx, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, + Context *on_finish) { + return new GetInfoRequest(image_ctx, mirror_image, promotion_state, + primary_mirror_uuid, on_finish); + } + + GetInfoRequest(librados::IoCtx& io_ctx, asio::ContextWQ *op_work_queue, + const std::string &image_id, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, Context *on_finish); + GetInfoRequest(ImageCtxT &image_ctx, cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + std::string* primary_mirror_uuid, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_MIRROR_IMAGE + * | + * (journal /--------/ \--------\ (snapshot + * mode) | | mode) + * v v + * GET_JOURNAL_TAG_OWNER GET_SNAPCONTEXT (skip if + * | | cached) + * | v + * | GET_SNAPSHOTS (skip if + * | | cached) + * \--------\ /--------/ + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx = nullptr; + librados::IoCtx &m_io_ctx; + asio::ContextWQ *m_op_work_queue; + std::string m_image_id; + cls::rbd::MirrorImage *m_mirror_image; + PromotionState *m_promotion_state; + std::string* m_primary_mirror_uuid; + Context *m_on_finish; + + CephContext *m_cct; + + bufferlist m_out_bl; + std::string m_mirror_uuid; + ::SnapContext m_snapc; + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void get_journal_tag_owner(); + void handle_get_journal_tag_owner(int r); + + void get_snapcontext(); + void handle_get_snapcontext(int r); + + void get_snapshots(); + void handle_get_snapshots(int r); + + void finish(int r); + + void calc_promotion_state( + const std::map &snap_info); +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::GetInfoRequest; + +#endif // CEPH_LIBRBD_MIRROR_GET_INFO_REQUEST_H + diff --git a/src/librbd/mirror/GetStatusRequest.cc b/src/librbd/mirror/GetStatusRequest.cc new file mode 100644 index 000000000..40d4a664b --- /dev/null +++ b/src/librbd/mirror/GetStatusRequest.cc @@ -0,0 +1,116 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/GetStatusRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/GetInfoRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::GetStatusRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void GetStatusRequest::send() { + *m_mirror_image_status = cls::rbd::MirrorImageStatus( + {{cls::rbd::MirrorImageSiteStatus::LOCAL_MIRROR_UUID, + cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN, "status not found"}}); + + get_info(); +} + +template +void GetStatusRequest::get_info() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + GetStatusRequest, &GetStatusRequest::handle_get_info>(this); + auto req = GetInfoRequest::create(m_image_ctx, m_mirror_image, + m_promotion_state, + &m_primary_mirror_uuid, ctx); + req->send(); +} + +template +void GetStatusRequest::handle_get_info(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + if (r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + } + finish(r); + return; + } else if (m_mirror_image->state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + finish(0); + return; + } + + get_status(); +} + +template +void GetStatusRequest::get_status() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_status_get_start( + &op, m_mirror_image->global_image_id); + + librados::AioCompletion *comp = create_rados_callback< + GetStatusRequest, &GetStatusRequest::handle_get_status>(this); + int r = m_image_ctx.md_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void GetStatusRequest::handle_get_status(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_image_status_get_finish(&iter, + m_mirror_image_status); + } + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirror image status: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void GetStatusRequest::finish(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::GetStatusRequest; diff --git a/src/librbd/mirror/GetStatusRequest.h b/src/librbd/mirror/GetStatusRequest.h new file mode 100644 index 000000000..581a0d667 --- /dev/null +++ b/src/librbd/mirror/GetStatusRequest.h @@ -0,0 +1,86 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_GET_STATUS_REQUEST_H +#define CEPH_LIBRBD_MIRROR_GET_STATUS_REQUEST_H + +#include "include/buffer.h" +#include "librbd/mirror/Types.h" +#include + +struct Context; +namespace cls { namespace rbd { struct MirrorImage; } } +namespace cls { namespace rbd { struct MirrorImageStatus; } } + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class GetStatusRequest { +public: + static GetStatusRequest *create(ImageCtxT &image_ctx, + cls::rbd::MirrorImageStatus *status, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, + Context *on_finish) { + return new GetStatusRequest(image_ctx, status, mirror_image, + promotion_state, on_finish); + } + + GetStatusRequest(ImageCtxT &image_ctx, cls::rbd::MirrorImageStatus *status, + cls::rbd::MirrorImage *mirror_image, + PromotionState *promotion_state, Context *on_finish) + : m_image_ctx(image_ctx), m_mirror_image_status(status), + m_mirror_image(mirror_image), m_promotion_state(promotion_state), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_INFO + * | + * v + * GET_STATUS + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + cls::rbd::MirrorImageStatus *m_mirror_image_status; + cls::rbd::MirrorImage *m_mirror_image; + PromotionState *m_promotion_state; + Context *m_on_finish; + + bufferlist m_out_bl; + std::string m_primary_mirror_uuid; + + void get_info(); + void handle_get_info(int r); + + void get_status(); + void handle_get_status(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::GetStatusRequest; + +#endif // CEPH_LIBRBD_MIRROR_GET_STATUS_REQUEST_H + diff --git a/src/librbd/mirror/GetUuidRequest.cc b/src/librbd/mirror/GetUuidRequest.cc new file mode 100644 index 000000000..f8209f905 --- /dev/null +++ b/src/librbd/mirror/GetUuidRequest.cc @@ -0,0 +1,86 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/GetUuidRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::GetUuidRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_rados_callback; + +template +GetUuidRequest::GetUuidRequest( + librados::IoCtx& io_ctx, std::string* mirror_uuid, Context* on_finish) + : m_mirror_uuid(mirror_uuid), m_on_finish(on_finish), + m_cct(reinterpret_cast(io_ctx.cct())) { + m_io_ctx.dup(io_ctx); + m_io_ctx.set_namespace(""); +} + +template +void GetUuidRequest::send() { + get_mirror_uuid(); +} + +template +void GetUuidRequest::get_mirror_uuid() { + ldout(m_cct, 20) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_uuid_get_start(&op); + + auto aio_comp = create_rados_callback< + GetUuidRequest, &GetUuidRequest::handle_get_mirror_uuid>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void GetUuidRequest::handle_get_mirror_uuid(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + if (r >= 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_uuid_get_finish(&it, m_mirror_uuid); + if (r >= 0 && m_mirror_uuid->empty()) { + r = -ENOENT; + } + } + + if (r < 0) { + if (r == -ENOENT) { + ldout(m_cct, 5) << "mirror uuid missing" << dendl; + } else { + lderr(m_cct) << "failed to retrieve mirror uuid: " << cpp_strerror(r) + << dendl; + } + *m_mirror_uuid = ""; + } + + finish(r); +} + +template +void GetUuidRequest::finish(int r) { + ldout(m_cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::GetUuidRequest; diff --git a/src/librbd/mirror/GetUuidRequest.h b/src/librbd/mirror/GetUuidRequest.h new file mode 100644 index 000000000..73cc2d5b2 --- /dev/null +++ b/src/librbd/mirror/GetUuidRequest.h @@ -0,0 +1,69 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_GET_UUID_REQUEST_H +#define CEPH_LIBRBD_MIRROR_GET_UUID_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class GetUuidRequest { +public: + static GetUuidRequest *create(librados::IoCtx& io_ctx, + std::string* mirror_uuid, Context* on_finish) { + return new GetUuidRequest(io_ctx, mirror_uuid, on_finish); + } + + GetUuidRequest(librados::IoCtx& io_ctx, std::string* mirror_uuid, + Context* on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_MIRROR_UUID + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx m_io_ctx; + std::string* m_mirror_uuid; + Context* m_on_finish; + + CephContext* m_cct; + + bufferlist m_out_bl; + + void get_mirror_uuid(); + void handle_get_mirror_uuid(int r); + + void finish(int r); +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::GetUuidRequest; + +#endif // CEPH_LIBRBD_MIRROR_GET_UUID_REQUEST_H diff --git a/src/librbd/mirror/ImageRemoveRequest.cc b/src/librbd/mirror/ImageRemoveRequest.cc new file mode 100644 index 000000000..1aa265dae --- /dev/null +++ b/src/librbd/mirror/ImageRemoveRequest.cc @@ -0,0 +1,98 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/ImageRemoveRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/MirroringWatcher.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::ImageRemoveRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using util::create_rados_callback; + +template +ImageRemoveRequest::ImageRemoveRequest( + librados::IoCtx& io_ctx, const std::string& global_image_id, + const std::string& image_id, Context* on_finish) + : m_io_ctx(io_ctx), m_global_image_id(global_image_id), m_image_id(image_id), + m_on_finish(on_finish), m_cct(static_cast(m_io_ctx.cct())) { +} + +template +void ImageRemoveRequest::send() { + remove_mirror_image(); +} + +template +void ImageRemoveRequest::remove_mirror_image() { + ldout(m_cct, 10) << dendl; + + librados::ObjectWriteOperation op; + cls_client::mirror_image_remove(&op, m_image_id); + + auto comp = create_rados_callback< + ImageRemoveRequest, + &ImageRemoveRequest::handle_remove_mirror_image>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void ImageRemoveRequest::handle_remove_mirror_image(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(m_cct) << "failed to remove mirroring image: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + notify_mirroring_watcher(); +} + +template +void ImageRemoveRequest::notify_mirroring_watcher() { + ldout(m_cct, 10) << dendl; + + auto ctx = util::create_context_callback< + ImageRemoveRequest, + &ImageRemoveRequest::handle_notify_mirroring_watcher>(this); + MirroringWatcher::notify_image_updated( + m_io_ctx, cls::rbd::MIRROR_IMAGE_STATE_DISABLED, + m_image_id, m_global_image_id, ctx); +} + +template +void ImageRemoveRequest::handle_notify_mirroring_watcher(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to notify mirror image update: " << cpp_strerror(r) + << dendl; + } + + finish(0); +} + +template +void ImageRemoveRequest::finish(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::ImageRemoveRequest; diff --git a/src/librbd/mirror/ImageRemoveRequest.h b/src/librbd/mirror/ImageRemoveRequest.h new file mode 100644 index 000000000..c04f9fadc --- /dev/null +++ b/src/librbd/mirror/ImageRemoveRequest.h @@ -0,0 +1,77 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_IMAGE_REMOVE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_IMAGE_REMOVE_REQUEST_H + +#include "include/rados/librados.hpp" +#include "common/ceph_mutex.h" +#include "cls/rbd/cls_rbd_types.h" + +#include + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace mirror { + +template +class ImageRemoveRequest { +public: + static ImageRemoveRequest *create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + const std::string& image_id, + Context* on_finish) { + return new ImageRemoveRequest(io_ctx, global_image_id, image_id, on_finish); + } + + ImageRemoveRequest(librados::IoCtx& io_ctx, + const std::string& global_image_id, + const std::string& image_id, + Context* on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * REMOVE_MIRROR_IMAGE + * | + * v + * NOTIFY_MIRRORING_WATCHER + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx& m_io_ctx; + std::string m_global_image_id; + std::string m_image_id; + Context* m_on_finish; + + CephContext* m_cct; + + void remove_mirror_image(); + void handle_remove_mirror_image(int r); + + void notify_mirroring_watcher(); + void handle_notify_mirroring_watcher(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::ImageRemoveRequest; + +#endif // CEPH_LIBRBD_MIRROR_IMAGE_REMOVE_REQUEST_H diff --git a/src/librbd/mirror/ImageStateUpdateRequest.cc b/src/librbd/mirror/ImageStateUpdateRequest.cc new file mode 100644 index 000000000..98e987190 --- /dev/null +++ b/src/librbd/mirror/ImageStateUpdateRequest.cc @@ -0,0 +1,151 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/ImageStateUpdateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/MirroringWatcher.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::ImageStateUpdateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using util::create_rados_callback; + +template +ImageStateUpdateRequest::ImageStateUpdateRequest( + librados::IoCtx& io_ctx, + const std::string& image_id, + cls::rbd::MirrorImageState mirror_image_state, + const cls::rbd::MirrorImage& mirror_image, + Context* on_finish) + : m_io_ctx(io_ctx), m_image_id(image_id), + m_mirror_image_state(mirror_image_state), m_mirror_image(mirror_image), + m_on_finish(on_finish), m_cct(static_cast(m_io_ctx.cct())) { + ceph_assert(m_mirror_image_state != cls::rbd::MIRROR_IMAGE_STATE_DISABLED); +} + +template +void ImageStateUpdateRequest::send() { + get_mirror_image(); +} + +template +void ImageStateUpdateRequest::get_mirror_image() { + if (!m_mirror_image.global_image_id.empty()) { + set_mirror_image(); + return; + } + + ldout(m_cct, 10) << dendl; + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_id); + + auto comp = create_rados_callback< + ImageStateUpdateRequest, + &ImageStateUpdateRequest::handle_get_mirror_image>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void ImageStateUpdateRequest::handle_get_mirror_image(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_image_get_finish(&iter, &m_mirror_image); + } + + if (r == -ENOENT) { + ldout(m_cct, 20) << "mirroring is disabled" << dendl; + finish(0); + return; + } else if (r < 0) { + lderr(m_cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + set_mirror_image(); +} + +template +void ImageStateUpdateRequest::set_mirror_image() { + if (m_mirror_image.state == m_mirror_image_state) { + finish(0); + return; + } + + ldout(m_cct, 10) << dendl; + m_mirror_image.state = m_mirror_image_state; + + librados::ObjectWriteOperation op; + cls_client::mirror_image_set(&op, m_image_id, m_mirror_image); + + auto comp = create_rados_callback< + ImageStateUpdateRequest, + &ImageStateUpdateRequest::handle_set_mirror_image>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void ImageStateUpdateRequest::handle_set_mirror_image(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to disable mirroring image: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + notify_mirroring_watcher(); +} + +template +void ImageStateUpdateRequest::notify_mirroring_watcher() { + ldout(m_cct, 10) << dendl; + + auto ctx = util::create_context_callback< + ImageStateUpdateRequest, + &ImageStateUpdateRequest::handle_notify_mirroring_watcher>(this); + MirroringWatcher::notify_image_updated( + m_io_ctx, m_mirror_image_state, m_image_id, m_mirror_image.global_image_id, + ctx); +} + +template +void ImageStateUpdateRequest::handle_notify_mirroring_watcher(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_cct) << "failed to notify mirror image update: " << cpp_strerror(r) + << dendl; + } + + finish(0); +} + +template +void ImageStateUpdateRequest::finish(int r) { + ldout(m_cct, 10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::ImageStateUpdateRequest; diff --git a/src/librbd/mirror/ImageStateUpdateRequest.h b/src/librbd/mirror/ImageStateUpdateRequest.h new file mode 100644 index 000000000..9e0affe6a --- /dev/null +++ b/src/librbd/mirror/ImageStateUpdateRequest.h @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_IMAGE_STATE_UPDATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_IMAGE_STATE_UPDATE_REQUEST_H + +#include "include/rados/librados.hpp" +#include "common/ceph_mutex.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" + +#include + +class Context; + +namespace librbd { + +class ImageCtx; + +namespace mirror { + +template +class ImageStateUpdateRequest { +public: + static ImageStateUpdateRequest *create( + librados::IoCtx& io_ctx, + const std::string& image_id, + cls::rbd::MirrorImageState mirror_image_state, + const cls::rbd::MirrorImage& mirror_image, + Context* on_finish) { + return new ImageStateUpdateRequest( + io_ctx, image_id, mirror_image_state, mirror_image, on_finish); + } + + ImageStateUpdateRequest( + librados::IoCtx& io_ctx, + const std::string& image_id, + cls::rbd::MirrorImageState mirror_image_state, + const cls::rbd::MirrorImage& mirror_image, + Context* on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v (skip if provided) + * GET_MIRROR_IMAGE + * | + * v + * SET_MIRROR_IMAGE + * | + * v + * NOTIFY_MIRRORING_WATCHER + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx& m_io_ctx; + std::string m_image_id; + cls::rbd::MirrorImageState m_mirror_image_state; + cls::rbd::MirrorImage m_mirror_image; + Context* m_on_finish; + + CephContext* m_cct; + bufferlist m_out_bl; + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void set_mirror_image(); + void handle_set_mirror_image(int r); + + void notify_mirroring_watcher(); + void handle_notify_mirroring_watcher(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::ImageStateUpdateRequest; + +#endif // CEPH_LIBRBD_MIRROR_IMAGE_STATE_UPDATE_REQUEST_H diff --git a/src/librbd/mirror/PromoteRequest.cc b/src/librbd/mirror/PromoteRequest.cc new file mode 100644 index 000000000..b119e4edc --- /dev/null +++ b/src/librbd/mirror/PromoteRequest.cc @@ -0,0 +1,115 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/PromoteRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "librbd/mirror/snapshot/PromoteRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::PromoteRequest: " << this \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { + +using librbd::util::create_context_callback; + +template +void PromoteRequest::send() { + get_info(); +} + +template +void PromoteRequest::get_info() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, &PromoteRequest::handle_get_info>(this); + auto req = GetInfoRequest::create(m_image_ctx, &m_mirror_image, + &m_promotion_state, + &m_primary_mirror_uuid, ctx); + req->send(); +} + +template +void PromoteRequest::handle_get_info(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else if (m_mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + lderr(cct) << "mirroring is not currently enabled" << dendl; + finish(-EINVAL); + return; + } else if (m_promotion_state == PROMOTION_STATE_PRIMARY) { + lderr(cct) << "image is already primary" << dendl; + finish(-EINVAL); + return; + } else if (m_promotion_state == PROMOTION_STATE_NON_PRIMARY && !m_force) { + lderr(cct) << "image is primary within a remote cluster or demotion is not propagated yet" + << dendl; + finish(-EBUSY); + return; + } + + promote(); +} + +template +void PromoteRequest::promote() { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, &PromoteRequest::handle_promote>(this); + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_JOURNAL) { + Journal::promote(&m_image_ctx, ctx); + } else if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + auto req = mirror::snapshot::PromoteRequest::create( + &m_image_ctx, m_mirror_image.global_image_id, ctx); + req->send(); + } else { + lderr(cct) << "unknown image mirror mode: " << m_mirror_image.mode << dendl; + finish(-EOPNOTSUPP); + } +} + +template +void PromoteRequest::handle_promote(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to promote image: " << cpp_strerror(r) + << dendl; + } + + finish(r); +} + +template +void PromoteRequest::finish(int r) { + CephContext *cct = m_image_ctx.cct; + ldout(cct, 20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::PromoteRequest; diff --git a/src/librbd/mirror/PromoteRequest.h b/src/librbd/mirror/PromoteRequest.h new file mode 100644 index 000000000..c54f3bb76 --- /dev/null +++ b/src/librbd/mirror/PromoteRequest.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_PROMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_PROMOTE_REQUEST_H + +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { + +template +class PromoteRequest { +public: + static PromoteRequest *create(ImageCtxT &image_ctx, bool force, + Context *on_finish) { + return new PromoteRequest(image_ctx, force, on_finish); + } + + PromoteRequest(ImageCtxT &image_ctx, bool force, Context *on_finish) + : m_image_ctx(image_ctx), m_force(force), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_INFO + * | + * v + * GET_TAG_OWNER + * | + * v + * PROMOTE + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT &m_image_ctx; + bool m_force; + Context *m_on_finish; + + cls::rbd::MirrorImage m_mirror_image; + PromotionState m_promotion_state = PROMOTION_STATE_PRIMARY; + std::string m_primary_mirror_uuid; + + void get_info(); + void handle_get_info(int r); + + void promote(); + void handle_promote(int r); + + void finish(int r); + +}; + +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::PromoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_PROMOTE_REQUEST_H diff --git a/src/librbd/mirror/Types.h b/src/librbd/mirror/Types.h new file mode 100644 index 000000000..2388b74ef --- /dev/null +++ b/src/librbd/mirror/Types.h @@ -0,0 +1,21 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_TYPES_H +#define CEPH_LIBRBD_MIRROR_TYPES_H + +namespace librbd { +namespace mirror { + +enum PromotionState { + PROMOTION_STATE_UNKNOWN, + PROMOTION_STATE_PRIMARY, + PROMOTION_STATE_NON_PRIMARY, + PROMOTION_STATE_ORPHAN +}; + +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_TYPES_H + diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc new file mode 100644 index 000000000..eed0aa506 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc @@ -0,0 +1,273 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Utils.h" +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::CreateNonPrimaryRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +CreateNonPrimaryRequest::CreateNonPrimaryRequest( + I* image_ctx, bool demoted, const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, const SnapSeqs& snap_seqs, + const ImageState &image_state, uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_demoted(demoted), + m_primary_mirror_uuid(primary_mirror_uuid), + m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs), + m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) { + m_default_ns_ctx.dup(m_image_ctx->md_ctx); + m_default_ns_ctx.set_namespace(""); +} + +template +void CreateNonPrimaryRequest::send() { + refresh_image(); +} + +template +void CreateNonPrimaryRequest::refresh_image() { + if (!m_image_ctx->state->is_refresh_required()) { + get_mirror_image(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template +void CreateNonPrimaryRequest::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_mirror_image(); +} + +template +void CreateNonPrimaryRequest::get_mirror_image() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_ctx->id); + + librados::AioCompletion *comp = create_rados_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_get_mirror_image>(this); + int r = m_image_ctx->md_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void CreateNonPrimaryRequest::handle_get_mirror_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + cls::rbd::MirrorImage mirror_image; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_image_get_finish(&iter, &mirror_image); + } + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + lderr(cct) << "snapshot based mirroring is not enabled" << dendl; + finish(-EINVAL); + return; + } + + if (!is_orphan() && !util::can_create_non_primary_snapshot(m_image_ctx)) { + finish(-EINVAL); + return; + } + + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_snap_name = ".mirror.non_primary." + mirror_image.global_image_id + "." + + uuid_gen.to_string(); + + get_mirror_peers(); +} + +template +void CreateNonPrimaryRequest::get_mirror_peers() { + if (!m_demoted) { + create_snapshot(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + auto aio_comp = create_rados_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_get_mirror_peers>(this); + m_out_bl.clear(); + int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void CreateNonPrimaryRequest::handle_get_mirror_peers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + std::vector peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + + create_snapshot(); +} + +template +void CreateNonPrimaryRequest::create_snapshot() { + CephContext *cct = m_image_ctx->cct; + + cls::rbd::MirrorSnapshotNamespace ns{ + (m_demoted ? cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED : + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY), {}, + m_primary_mirror_uuid, m_primary_snap_id}; + if (m_demoted) { + ns.mirror_peer_uuids = m_mirror_peer_uuids; + } + ns.snap_seqs = m_snap_seqs; + ns.complete = is_orphan(); + ldout(cct, 15) << "ns=" << ns << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_create_snapshot>(this); + m_image_ctx->operations->snap_create(ns, m_snap_name, 0, m_prog_ctx, ctx); +} + +template +void CreateNonPrimaryRequest::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + write_image_state(); +} + +template +void CreateNonPrimaryRequest::write_image_state() { + uint64_t snap_id; + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + snap_id = m_image_ctx->get_snap_id( + cls::rbd::MirrorSnapshotNamespace{}, m_snap_name); + } + + if (m_snap_id != nullptr) { + *m_snap_id = snap_id; + } + + if (is_orphan()) { + finish(0); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_write_image_state>(this); + + auto req = WriteImageStateRequest::create(m_image_ctx, snap_id, + m_image_state, ctx); + req->send(); +} + +template +void CreateNonPrimaryRequest::handle_write_image_state(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write image state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void CreateNonPrimaryRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::CreateNonPrimaryRequest; diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h new file mode 100644 index 000000000..36f155413 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h @@ -0,0 +1,123 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H + +#include "include/buffer.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/Types.h" +#include "librbd/internal.h" +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class CreateNonPrimaryRequest { +public: + static CreateNonPrimaryRequest *create(ImageCtxT *image_ctx, + bool demoted, + const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, + const SnapSeqs& snap_seqs, + const ImageState &image_state, + uint64_t *snap_id, + Context *on_finish) { + return new CreateNonPrimaryRequest(image_ctx, demoted, primary_mirror_uuid, + primary_snap_id, snap_seqs, image_state, + snap_id, on_finish); + } + + CreateNonPrimaryRequest(ImageCtxT *image_ctx, + bool demoted, + const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, + const SnapSeqs& snap_seqs, + const ImageState &image_state, uint64_t *snap_id, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * REFRESH_IMAGE + * | + * v + * GET_MIRROR_IMAGE + * | + * v (skip if not needed) + * GET_MIRROR_PEERS + * | + * v + * CREATE_SNAPSHOT + * | + * v + * WRITE_IMAGE_STATE + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + bool m_demoted; + std::string m_primary_mirror_uuid; + uint64_t m_primary_snap_id; + SnapSeqs m_snap_seqs; + ImageState m_image_state; + uint64_t *m_snap_id; + Context *m_on_finish; + + librados::IoCtx m_default_ns_ctx; + std::set m_mirror_peer_uuids; + + std::string m_snap_name; + + bufferlist m_out_bl; + NoOpProgressContext m_prog_ctx; + + bool is_orphan() const { + return m_primary_mirror_uuid.empty(); + } + + void refresh_image(); + void handle_refresh_image(int r); + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void get_mirror_peers(); + void handle_get_mirror_peers(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void write_image_state(); + void handle_write_image_state(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::CreateNonPrimaryRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc new file mode 100644 index 000000000..c8e3a4fe7 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc @@ -0,0 +1,281 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::CreatePrimaryRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +CreatePrimaryRequest::CreatePrimaryRequest( + I *image_ctx, const std::string& global_image_id, + uint64_t clean_since_snap_id, uint64_t snap_create_flags, uint32_t flags, + uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_clean_since_snap_id(clean_since_snap_id), + m_snap_create_flags(snap_create_flags), m_flags(flags), m_snap_id(snap_id), + m_on_finish(on_finish) { + m_default_ns_ctx.dup(m_image_ctx->md_ctx); + m_default_ns_ctx.set_namespace(""); +} + +template +void CreatePrimaryRequest::send() { + if (!util::can_create_primary_snapshot( + m_image_ctx, + ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0), + ((m_flags & CREATE_PRIMARY_FLAG_FORCE) != 0), nullptr, nullptr)) { + finish(-EINVAL); + return; + } + + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_snap_name = ".mirror.primary." + m_global_image_id + "." + + uuid_gen.to_string(); + + get_mirror_peers(); +} + +template +void CreatePrimaryRequest::get_mirror_peers() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + librados::AioCompletion *comp = create_rados_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_get_mirror_peers>(this); + m_out_bl.clear(); + int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void CreatePrimaryRequest::handle_get_mirror_peers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + std::vector peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + + if (m_mirror_peer_uuids.empty() && + ((m_flags & CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS) == 0)) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + finish(-EINVAL); + return; + } + + create_snapshot(); +} + +template +void CreatePrimaryRequest::create_snapshot() { + cls::rbd::MirrorSnapshotNamespace ns{ + ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0 ? + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED : + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY), + m_mirror_peer_uuids, "", m_clean_since_snap_id}; + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "name=" << m_snap_name << ", " + << "ns=" << ns << dendl; + auto ctx = create_context_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_create_snapshot>(this); + m_image_ctx->operations->snap_create(ns, m_snap_name, m_snap_create_flags, + m_prog_ctx, ctx); +} + +template +void CreatePrimaryRequest::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + refresh_image(); +} + +template +void CreatePrimaryRequest::refresh_image() { + // refresh is required to retrieve the snapshot id (if snapshot + // created via remote RPC) and complete flag (regardless) + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template +void CreatePrimaryRequest::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (m_snap_id != nullptr) { + std::shared_lock image_locker{m_image_ctx->image_lock}; + *m_snap_id = m_image_ctx->get_snap_id( + cls::rbd::MirrorSnapshotNamespace{}, m_snap_name); + ldout(cct, 15) << "snap_id=" << *m_snap_id << dendl; + } + + unlink_peer(); +} + +template +void CreatePrimaryRequest::unlink_peer() { + // TODO: Document semantics for unlink_peer + uint64_t max_snapshots = m_image_ctx->config.template get_val( + "rbd_mirroring_max_mirroring_snapshots"); + ceph_assert(max_snapshots >= 3); + + std::string peer_uuid; + uint64_t snap_id = CEPH_NOSNAP; + + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + for (const auto& peer : m_mirror_peer_uuids) { + for (const auto& snap_info_pair : m_image_ctx->snap_info) { + auto info = std::get_if( + &snap_info_pair.second.snap_namespace); + if (info == nullptr) { + continue; + } + if (info->mirror_peer_uuids.empty() || + (info->mirror_peer_uuids.count(peer) != 0 && + info->is_primary() && !info->complete)) { + peer_uuid = peer; + snap_id = snap_info_pair.first; + goto do_unlink; + } + } + } + for (const auto& peer : m_mirror_peer_uuids) { + size_t count = 0; + uint64_t unlink_snap_id = 0; + for (const auto& snap_info_pair : m_image_ctx->snap_info) { + auto info = std::get_if( + &snap_info_pair.second.snap_namespace); + if (info == nullptr) { + continue; + } + if (info->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + // reset counters -- we count primary snapshots after the last + // promotion + count = 0; + unlink_snap_id = 0; + continue; + } + if (info->mirror_peer_uuids.count(peer) == 0) { + // snapshot is not linked with this peer + continue; + } + count++; + if (count == max_snapshots) { + unlink_snap_id = snap_info_pair.first; + } + if (count > max_snapshots) { + peer_uuid = peer; + snap_id = unlink_snap_id; + goto do_unlink; + } + } + } + } + + finish(0); + return; + +do_unlink: + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "peer=" << peer_uuid << ", snap_id=" << snap_id << dendl; + + auto ctx = create_context_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_unlink_peer>(this); + auto req = UnlinkPeerRequest::create(m_image_ctx, snap_id, peer_uuid, true, + ctx); + req->send(); +} + +template +void CreatePrimaryRequest::handle_unlink_peer(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to unlink peer: " << cpp_strerror(r) << dendl; + finish(0); // not fatal + return; + } + + unlink_peer(); +} + +template +void CreatePrimaryRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::CreatePrimaryRequest; diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.h b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h new file mode 100644 index 000000000..b8e84cf2b --- /dev/null +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h @@ -0,0 +1,106 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/internal.h" +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class CreatePrimaryRequest { +public: + static CreatePrimaryRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + uint64_t clean_since_snap_id, + uint64_t snap_create_flags, + uint32_t flags, uint64_t *snap_id, + Context *on_finish) { + return new CreatePrimaryRequest(image_ctx, global_image_id, + clean_since_snap_id, snap_create_flags, flags, + snap_id, on_finish); + } + + CreatePrimaryRequest(ImageCtxT *image_ctx, + const std::string& global_image_id, + uint64_t clean_since_snap_id, uint64_t snap_create_flags, + uint32_t flags, uint64_t *snap_id, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_MIRROR_PEERS + * | + * v + * CREATE_SNAPSHOT + * | + * v + * REFRESH_IMAGE + * | + * v + * UNLINK_PEER (skip if not needed, + * | repeat if needed) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + uint64_t m_clean_since_snap_id; + const uint64_t m_snap_create_flags; + const uint32_t m_flags; + uint64_t *m_snap_id; + Context *m_on_finish; + + librados::IoCtx m_default_ns_ctx; + std::set m_mirror_peer_uuids; + std::string m_snap_name; + + bufferlist m_out_bl; + NoOpProgressContext m_prog_ctx; + + void get_mirror_peers(); + void handle_get_mirror_peers(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void refresh_image(); + void handle_refresh_image(int r); + + void unlink_peer(); + void handle_unlink_peer(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::CreatePrimaryRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H diff --git a/src/librbd/mirror/snapshot/DemoteRequest.cc b/src/librbd/mirror/snapshot/DemoteRequest.cc new file mode 100644 index 000000000..ccaa33c83 --- /dev/null +++ b/src/librbd/mirror/snapshot/DemoteRequest.cc @@ -0,0 +1,110 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/DemoteRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::DemoteRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void DemoteRequest::send() { + enable_non_primary_feature(); +} + +template +void DemoteRequest::enable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // ensure image is flagged with non-primary feature so that + // standard RBD clients cannot write to it. + librados::ObjectWriteOperation op; + cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY, + RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + DemoteRequest, + &DemoteRequest::handle_enable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void DemoteRequest::handle_enable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to enable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_snapshot(); +} + +template +void DemoteRequest::create_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_create_snapshot>(this); + + auto req = CreatePrimaryRequest::create( + m_image_ctx, m_global_image_id, CEPH_NOSNAP, + SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, + (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | + snapshot::CREATE_PRIMARY_FLAG_DEMOTED), nullptr, ctx); + req->send(); +} + +template +void DemoteRequest::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void DemoteRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::DemoteRequest; diff --git a/src/librbd/mirror/snapshot/DemoteRequest.h b/src/librbd/mirror/snapshot/DemoteRequest.h new file mode 100644 index 000000000..63c935645 --- /dev/null +++ b/src/librbd/mirror/snapshot/DemoteRequest.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H + +#include "include/buffer.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class DemoteRequest { +public: + static DemoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + Context *on_finish) { + return new DemoteRequest(image_ctx, global_image_id, on_finish); + } + + DemoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * ENABLE_NON_PRIMARY_FEATURE + * | + * v + * CREATE_SNAPSHOT + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + Context *m_on_finish; + + void enable_non_primary_feature(); + void handle_enable_non_primary_feature(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void finish(int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::DemoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/GetImageStateRequest.cc b/src/librbd/mirror/snapshot/GetImageStateRequest.cc new file mode 100644 index 000000000..4692f88cb --- /dev/null +++ b/src/librbd/mirror/snapshot/GetImageStateRequest.cc @@ -0,0 +1,114 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/GetImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Types.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::GetImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + +template +void GetImageStateRequest::send() { + read_object(); +} + + +template +void GetImageStateRequest::read_object() { + CephContext *cct = m_image_ctx->cct; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_index); + ldout(cct, 15) << oid << dendl; + + librados::ObjectReadOperation op; + m_bl.clear(); + op.read(0, 0, &m_bl, nullptr); + + librados::AioCompletion *comp = create_rados_callback< + GetImageStateRequest, + &GetImageStateRequest::handle_read_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op, nullptr); + ceph_assert(r == 0); + comp->release(); +} + +template +void GetImageStateRequest::handle_read_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to read image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + auto iter = m_bl.cbegin(); + + if (m_object_index == 0) { + ImageStateHeader header; + try { + using ceph::decode; + decode(header, iter); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state object header" << dendl; + finish(-EBADMSG); + return; + } + m_object_count = header.object_count; + } + + bufferlist bl; + bl.substr_of(m_bl, iter.get_off(), m_bl.length() - iter.get_off()); + m_state_bl.claim_append(bl); + + m_object_index++; + + if (m_object_index >= m_object_count) { + finish(0); + return; + } + + read_object(); +} + +template +void GetImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + try { + using ceph::decode; + decode(*m_image_state, m_state_bl); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state" << dendl; + r = -EBADMSG; + } + } + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::GetImageStateRequest; diff --git a/src/librbd/mirror/snapshot/GetImageStateRequest.h b/src/librbd/mirror/snapshot/GetImageStateRequest.h new file mode 100644 index 000000000..483e3a228 --- /dev/null +++ b/src/librbd/mirror/snapshot/GetImageStateRequest.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H + +#include "include/buffer.h" +#include "include/types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +struct ImageState; + +template +class GetImageStateRequest { +public: + static GetImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + ImageState *image_state, + Context *on_finish) { + return new GetImageStateRequest(image_ctx, snap_id, image_state, on_finish); + } + + GetImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + ImageState *image_state, Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_image_state(image_state), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * READ_OBJECT (repeat for + * | every object) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + ImageState *m_image_state; + Context *m_on_finish; + + bufferlist m_bl; + bufferlist m_state_bl; + + size_t m_object_count = 0; + size_t m_object_index = 0; + + void read_object(); + void handle_read_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::GetImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/ImageMeta.cc b/src/librbd/mirror/snapshot/ImageMeta.cc new file mode 100644 index 000000000..826899775 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.cc @@ -0,0 +1,175 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "json_spirit/json_spirit.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/WatchNotifyTypes.h" +#include "librbd/mirror/snapshot/Utils.h" +#include "librbd/watcher/Notifier.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::ImageMeta: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; +using librbd::mirror::snapshot::util::get_image_meta_key; + +template +ImageMeta::ImageMeta(I* image_ctx, const std::string& mirror_uuid) + : m_image_ctx(image_ctx), m_mirror_uuid(mirror_uuid) { +} + +template +void ImageMeta::load(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + librados::ObjectReadOperation op; + cls_client::metadata_get_start(&op, get_image_meta_key(m_mirror_uuid)); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_load(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_load(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + std::string data; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = cls_client::metadata_get_finish(&it, &data); + } + + if (r == -ENOENT) { + ldout(m_image_ctx->cct, 15) << "no snapshot-based mirroring image-meta: " + << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } else if (r < 0) { + lderr(m_image_ctx->cct) << "failed to load snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + bool json_valid = false; + json_spirit::mValue json_root; + if (json_spirit::read(data, json_root)) { + try { + auto& json_obj = json_root.get_obj(); + resync_requested = json_obj["resync_requested"].get_bool(); + json_valid = true; + } catch (std::runtime_error&) { + } + } + + if (!json_valid) { + lderr(m_image_ctx->cct) << "invalid image-meta JSON received" << dendl; + on_finish->complete(-EBADMSG); + return; + } + + on_finish->complete(0); +} + +template +void ImageMeta::save(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + // simple implementation for now + std::string json = "{\"resync_requested\": " + + std::string(resync_requested ? "true" : "false") + "}"; + + bufferlist bl; + bl.append(json); + + // avoid using built-in metadata_set operation since that would require + // opening the non-primary image in read/write mode which isn't supported + librados::ObjectWriteOperation op; + cls_client::metadata_set(&op, {{get_image_meta_key(m_mirror_uuid), bl}}); + + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_save(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_save(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to save snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + notify_update(on_finish); +} + +template +void ImageMeta::notify_update(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << dendl; + + // directly send header notification on image since you cannot + // open a non-primary image read/write and therefore cannot re-use + // the ImageWatcher to send the notification + bufferlist bl; + encode(watch_notify::NotifyMessage(new watch_notify::HeaderUpdatePayload()), + bl); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_notify_update(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_notify( + m_image_ctx->header_oid, aio_comp, bl, watcher::Notifier::NOTIFY_TIMEOUT, + &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_notify_update(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to notify image update: " + << cpp_strerror(r) << dendl; + } + on_finish->complete(r); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::ImageMeta; diff --git a/src/librbd/mirror/snapshot/ImageMeta.h b/src/librbd/mirror/snapshot/ImageMeta.h new file mode 100644 index 000000000..5d05f1927 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.h @@ -0,0 +1,78 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H + +#include "include/rados/librados.hpp" +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class ImageMeta { +public: + static ImageMeta* create(ImageCtxT* image_ctx, + const std::string& mirror_uuid) { + return new ImageMeta(image_ctx, mirror_uuid); + } + + ImageMeta(ImageCtxT* image_ctx, const std::string& mirror_uuid); + + void load(Context* on_finish); + void save(Context* on_finish); + + bool resync_requested = false; + +private: + /** + * @verbatim + * + * + * | + * v + * METADATA_GET + * | + * v + * + * | + * v + * METADATA_SET + * | + * v + * NOTIFY_UPDATE + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT* m_image_ctx; + std::string m_mirror_uuid; + + bufferlist m_out_bl; + + void handle_load(Context* on_finish, int r); + + void handle_save(Context* on_finish, int r); + + void notify_update(Context* on_finish); + void handle_notify_update(Context* on_finish, int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::ImageMeta; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc new file mode 100644 index 000000000..9718c299e --- /dev/null +++ b/src/librbd/mirror/snapshot/PromoteRequest.cc @@ -0,0 +1,405 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/PromoteRequest.h" +#include "common/Timer.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/image/ListWatchersRequest.h" +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::PromoteRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void PromoteRequest::send() { + CephContext *cct = m_image_ctx->cct; + bool requires_orphan = false; + if (!util::can_create_primary_snapshot(m_image_ctx, false, true, + &requires_orphan, + &m_rollback_snap_id)) { + lderr(cct) << "cannot promote" << dendl; + finish(-EINVAL); + return; + } else if (m_rollback_snap_id == CEPH_NOSNAP && !requires_orphan) { + create_promote_snapshot(); + return; + } + + ldout(cct, 15) << "requires_orphan=" << requires_orphan << ", " + << "rollback_snap_id=" << m_rollback_snap_id << dendl; + create_orphan_snapshot(); +} + +template +void PromoteRequest::create_orphan_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_create_orphan_snapshot>(this); + + auto req = CreateNonPrimaryRequest::create( + m_image_ctx, false, "", CEPH_NOSNAP, {}, {}, nullptr, ctx); + req->send(); +} + +template +void PromoteRequest::handle_create_orphan_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create orphan snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + list_watchers(); +} + +template +void PromoteRequest::list_watchers() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_list_watchers>(this); + + m_watchers.clear(); + auto flags = librbd::image::LIST_WATCHERS_FILTER_OUT_MY_INSTANCE | + librbd::image::LIST_WATCHERS_MIRROR_INSTANCES_ONLY; + auto req = librbd::image::ListWatchersRequest::create( + *m_image_ctx, flags, &m_watchers, ctx); + req->send(); +} + +template +void PromoteRequest::handle_list_watchers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to list watchers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_watchers.empty()) { + acquire_exclusive_lock(); + return; + } + + wait_update_notify(); +} + +template +void PromoteRequest::wait_update_notify() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + ImageCtx::get_timer_instance(cct, &m_timer, &m_timer_lock); + + std::lock_guard timer_lock{*m_timer_lock}; + + m_scheduler_ticks = 5; + + int r = m_image_ctx->state->register_update_watcher(&m_update_watch_ctx, + &m_update_watcher_handle); + if (r < 0) { + lderr(cct) << "failed to register update watcher: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + scheduler_unregister_update_watcher(); +} + +template +void PromoteRequest::handle_update_notify() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + std::lock_guard timer_lock{*m_timer_lock}; + m_scheduler_ticks = 0; +} + +template +void PromoteRequest::scheduler_unregister_update_watcher() { + ceph_assert(ceph_mutex_is_locked(*m_timer_lock)); + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "scheduler_ticks=" << m_scheduler_ticks << dendl; + + if (m_scheduler_ticks > 0) { + m_scheduler_ticks--; + m_timer->add_event_after(1, new LambdaContext([this](int) { + scheduler_unregister_update_watcher(); + })); + return; + } + + m_image_ctx->op_work_queue->queue(new LambdaContext([this](int) { + unregister_update_watcher(); + }), 0); +} + +template +void PromoteRequest::unregister_update_watcher() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_unregister_update_watcher>(this); + + m_image_ctx->state->unregister_update_watcher(m_update_watcher_handle, ctx); +} + +template +void PromoteRequest::handle_unregister_update_watcher(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to unregister update watcher: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + list_watchers(); +} + +template +void PromoteRequest::acquire_exclusive_lock() { + { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr && + !m_image_ctx->exclusive_lock->is_lock_owner()) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + m_lock_acquired = true; + m_image_ctx->exclusive_lock->block_requests(0); + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_acquire_exclusive_lock>(this); + + m_image_ctx->exclusive_lock->acquire_lock(ctx); + return; + } + } + + rollback(); +} + +template +void PromoteRequest::handle_acquire_exclusive_lock(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to acquire exclusive lock: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr && + !m_image_ctx->exclusive_lock->is_lock_owner()) { + lderr(cct) << "failed to acquire exclusive lock" << dendl; + r = m_image_ctx->exclusive_lock->get_unlocked_op_error(); + locker.unlock(); + finish(r); + return; + } + } + + rollback(); +} + +template +void PromoteRequest::rollback() { + if (m_rollback_snap_id == CEPH_NOSNAP) { + create_promote_snapshot(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + std::shared_lock owner_locker{m_image_ctx->owner_lock}; + std::shared_lock image_locker{m_image_ctx->image_lock}; + + auto info = m_image_ctx->get_snap_info(m_rollback_snap_id); + ceph_assert(info != nullptr); + auto snap_namespace = info->snap_namespace; + auto snap_name = info->name; + + image_locker.unlock(); + + auto ctx = create_async_context_callback( + *m_image_ctx, create_context_callback< + PromoteRequest, &PromoteRequest::handle_rollback>(this)); + + m_image_ctx->operations->execute_snap_rollback(snap_namespace, snap_name, + m_progress_ctx, ctx); +} + +template +void PromoteRequest::handle_rollback(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to rollback: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_promote_snapshot(); +} + +template +void PromoteRequest::create_promote_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_create_promote_snapshot>(this); + + auto req = CreatePrimaryRequest::create( + m_image_ctx, m_global_image_id, CEPH_NOSNAP, + SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, + (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | + snapshot::CREATE_PRIMARY_FLAG_FORCE), nullptr, ctx); + req->send(); +} + +template +void PromoteRequest::handle_create_promote_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create promote snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + disable_non_primary_feature(); +} + +template +void PromoteRequest::disable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // remove the non-primary feature flag so that the image can be + // R/W by standard RBD clients + librados::ObjectWriteOperation op; + cls_client::set_features(&op, 0U, RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + PromoteRequest, + &PromoteRequest::handle_disable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void PromoteRequest::handle_disable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to disable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + release_exclusive_lock(); +} + +template +void PromoteRequest::release_exclusive_lock() { + if (m_lock_acquired) { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + m_image_ctx->exclusive_lock->unblock_requests(); + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_release_exclusive_lock>(this); + + m_image_ctx->exclusive_lock->release_lock(ctx); + return; + } + } + + finish(0); +} + +template +void PromoteRequest::handle_release_exclusive_lock(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to release exclusive lock: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void PromoteRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::PromoteRequest; diff --git a/src/librbd/mirror/snapshot/PromoteRequest.h b/src/librbd/mirror/snapshot/PromoteRequest.h new file mode 100644 index 000000000..1d9a862a0 --- /dev/null +++ b/src/librbd/mirror/snapshot/PromoteRequest.h @@ -0,0 +1,151 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H + +#include "include/buffer.h" +#include "include/rbd/librbd.hpp" +#include "common/ceph_mutex.h" +#include "common/Timer.h" +#include "librbd/internal.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class PromoteRequest { +public: + static PromoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + Context *on_finish) { + return new PromoteRequest(image_ctx, global_image_id, on_finish); + } + + PromoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | (can promote) + * |\----------------------------------------\ + * | | + * | | + * v (skip if not needed) | + * CREATE_ORPHAN_SNAPSHOT | + * | | + * | /-- UNREGISTER_UPDATE_WATCHER <-\ | + * v v | | + * LIST_WATCHERS ----> WAIT_UPDATE_NOTIFY --/ | + * | | + * | (no watchers) | + * v | + * ACQUIRE_EXCLUSIVE_LOCK | + * | (skip if not needed) | + * v | + * ROLLBACK | + * | | + * v | + * CREATE_PROMOTE_SNAPSHOT <--------------------/ + * | + * v + * DISABLE_NON_PRIMARY_FEATURE + * | + * v + * RELEASE_EXCLUSIVE_LOCK (skip if not needed) + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + Context *m_on_finish; + + uint64_t m_rollback_snap_id = CEPH_NOSNAP; + bool m_lock_acquired = false; + NoOpProgressContext m_progress_ctx; + + class UpdateWatchCtx : public librbd::UpdateWatchCtx { + public: + UpdateWatchCtx(PromoteRequest *promote_request) + : promote_request(promote_request) { + } + + void handle_notify() { + promote_request->handle_update_notify(); + } + + private: + PromoteRequest *promote_request; + + } m_update_watch_ctx = {this}; + + std::list m_watchers; + uint64_t m_update_watcher_handle = 0; + uint64_t m_scheduler_ticks = 0; + SafeTimer *m_timer = nullptr; + ceph::mutex *m_timer_lock = nullptr; + + void refresh_image(); + void handle_refresh_image(int r); + + void create_orphan_snapshot(); + void handle_create_orphan_snapshot(int r); + + void list_watchers(); + void handle_list_watchers(int r); + + void wait_update_notify(); + void handle_update_notify(); + void scheduler_unregister_update_watcher(); + + void unregister_update_watcher(); + void handle_unregister_update_watcher(int r); + + void acquire_exclusive_lock(); + void handle_acquire_exclusive_lock(int r); + + void rollback(); + void handle_rollback(int r); + + void create_promote_snapshot(); + void handle_create_promote_snapshot(int r); + + void disable_non_primary_feature(); + void handle_disable_non_primary_feature(int r); + + void release_exclusive_lock(); + void handle_release_exclusive_lock(int r); + + void finish(int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::PromoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc new file mode 100644 index 000000000..204e0489a --- /dev/null +++ b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc @@ -0,0 +1,131 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/RemoveImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Types.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::RemoveImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + +template +void RemoveImageStateRequest::send() { + get_object_count(); +} + + +template +void RemoveImageStateRequest::get_object_count() { + CephContext *cct = m_image_ctx->cct; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, 0); + ldout(cct, 15) << oid << dendl; + + librados::ObjectReadOperation op; + op.read(0, 0, &m_bl, nullptr); + + librados::AioCompletion *comp = create_rados_callback< + RemoveImageStateRequest, + &RemoveImageStateRequest::handle_get_object_count>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op, nullptr); + ceph_assert(r == 0); + comp->release(); +} + +template +void RemoveImageStateRequest::handle_get_object_count(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to read image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ImageStateHeader header(1); + auto iter = m_bl.cbegin(); + try { + using ceph::decode; + + decode(header, iter); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state object header" << dendl; + // still try to remove it + } + + m_object_count = header.object_count > 0 ? header.object_count : 1; + + remove_object(); +} + +template +void RemoveImageStateRequest::remove_object() { + CephContext *cct = m_image_ctx->cct; + + ceph_assert(m_object_count > 0); + m_object_count--; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_count); + ldout(cct, 15) << oid << dendl; + + librados::ObjectWriteOperation op; + op.remove(); + + librados::AioCompletion *comp = create_rados_callback< + RemoveImageStateRequest, + &RemoveImageStateRequest::handle_remove_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void RemoveImageStateRequest::handle_remove_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_object_count == 0) { + finish(0); + return; + } + + remove_object(); +} + +template +void RemoveImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::RemoveImageStateRequest; diff --git a/src/librbd/mirror/snapshot/RemoveImageStateRequest.h b/src/librbd/mirror/snapshot/RemoveImageStateRequest.h new file mode 100644 index 000000000..be7dad8e0 --- /dev/null +++ b/src/librbd/mirror/snapshot/RemoveImageStateRequest.h @@ -0,0 +1,75 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H + +#include "include/buffer.h" +#include "include/types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class RemoveImageStateRequest { +public: + static RemoveImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) { + return new RemoveImageStateRequest(image_ctx, snap_id, on_finish); + } + + RemoveImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_OBJECT_COUNT + * | + * v + * REMOVE_OBJECT (repeat for + * | every object) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + Context *m_on_finish; + + bufferlist m_bl; + + size_t m_object_count = 0; + + void get_object_count(); + void handle_get_object_count(int r); + + void remove_object(); + void handle_remove_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::RemoveImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/SetImageStateRequest.cc b/src/librbd/mirror/snapshot/SetImageStateRequest.cc new file mode 100644 index 000000000..9fcee0322 --- /dev/null +++ b/src/librbd/mirror/snapshot/SetImageStateRequest.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/SetImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/image/GetMetadataRequest.h" +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" + +#include + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror_snapshot::SetImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void SetImageStateRequest::send() { + get_name(); +} + +template +void SetImageStateRequest::get_name() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::dir_get_name_start(&op, m_image_ctx->id); + + librados::AioCompletion *comp = create_rados_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_get_name>(this); + m_bl.clear(); + int r = m_image_ctx->md_ctx.aio_operate(RBD_DIRECTORY, comp, &op, &m_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void SetImageStateRequest::handle_get_name(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + auto it = m_bl.cbegin(); + r = cls_client::dir_get_name_finish(&it, &m_image_state.name); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve image name: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ldout(cct, 15) << "name=" << m_image_state.name << dendl; + + get_snap_limit(); +} + +template +void SetImageStateRequest::get_snap_limit() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::snapshot_get_limit_start(&op); + + librados::AioCompletion *comp = create_rados_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_get_snap_limit>(this); + m_bl.clear(); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, comp, &op, + &m_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void SetImageStateRequest::handle_get_snap_limit(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + auto it = m_bl.cbegin(); + r = cls_client::snapshot_get_limit_finish(&it, &m_image_state.snap_limit); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve snapshot limit: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ldout(cct, 15) << "snap_limit=" << m_image_state.snap_limit << dendl; + + get_metadata(); +} + +template +void SetImageStateRequest::get_metadata() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_get_metadata>(this); + auto req = image::GetMetadataRequest::create( + m_image_ctx->md_ctx, m_image_ctx->header_oid, true, "", "", 0, + &m_image_state.metadata, ctx); + req->send(); +} + +template +void SetImageStateRequest::handle_get_metadata(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to retrieve metadata: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + + m_image_state.features = + m_image_ctx->features & ~RBD_FEATURES_IMPLICIT_ENABLE; + + for (auto &[snap_id, snap_info] : m_image_ctx->snap_info) { + auto type = cls::rbd::get_snap_namespace_type(snap_info.snap_namespace); + if (type != cls::rbd::SNAPSHOT_NAMESPACE_TYPE_USER) { + // only replicate user snapshots -- trash snapshots will be + // replicated by an implicit delete if required + continue; + } + m_image_state.snapshots[snap_id] = {snap_info.snap_namespace, + snap_info.name, + snap_info.protection_status}; + } + } + + write_image_state(); +} + +template +void SetImageStateRequest::write_image_state() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_write_image_state>(this); + + auto req = WriteImageStateRequest::create(m_image_ctx, m_snap_id, + m_image_state, ctx); + req->send(); +} + +template +void SetImageStateRequest::handle_write_image_state(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write image state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + update_primary_snapshot(); +} + +template +void SetImageStateRequest::update_primary_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_snapshot_set_copy_progress( + &op, m_snap_id, true, 0); + + auto aio_comp = create_rados_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_update_primary_snapshot>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void SetImageStateRequest::handle_update_primary_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to update primary snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void SetImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::SetImageStateRequest; diff --git a/src/librbd/mirror/snapshot/SetImageStateRequest.h b/src/librbd/mirror/snapshot/SetImageStateRequest.h new file mode 100644 index 000000000..fd2815494 --- /dev/null +++ b/src/librbd/mirror/snapshot/SetImageStateRequest.h @@ -0,0 +1,96 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H + +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class SetImageStateRequest { +public: + static SetImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) { + return new SetImageStateRequest(image_ctx, snap_id, on_finish); + } + + SetImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_NAME + * | + * v + * GET_SNAP_LIMIT + * | + * v + * GET_METADATA + * | + * v + * WRITE_IMAGE_STATE + * | + * v + * UPDATE_PRIMARY_SNAPSHOT + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + Context *m_on_finish; + + ImageState m_image_state; + + bufferlist m_bl; + bufferlist m_state_bl; + + void get_name(); + void handle_get_name(int r); + + void get_snap_limit(); + void handle_get_snap_limit(int r); + + void get_metadata(); + void handle_get_metadata(int r); + + void write_image_state(); + void handle_write_image_state(int r); + + void update_primary_snapshot(); + void handle_update_primary_snapshot(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::SetImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/Types.cc b/src/librbd/mirror/snapshot/Types.cc new file mode 100644 index 000000000..866b4c3e2 --- /dev/null +++ b/src/librbd/mirror/snapshot/Types.cc @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/Formatter.h" +#include "include/encoding.h" +#include "include/stringify.h" +#include "librbd/mirror/snapshot/Types.h" + +namespace librbd { +namespace mirror { +namespace snapshot { + +void ImageStateHeader::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(object_count, bl); + ENCODE_FINISH(bl); +} + +void ImageStateHeader::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(object_count, bl); + DECODE_FINISH(bl); +} + +void SnapState::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(snap_namespace, bl); + encode(name, bl); + encode(protection_status, bl); + ENCODE_FINISH(bl); +} + +void SnapState::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(snap_namespace, bl); + decode(name, bl); + decode(protection_status, bl); + DECODE_FINISH(bl); +} + +void SnapState::dump(Formatter *f) const { + f->open_object_section("namespace"); + snap_namespace.dump(f); + f->close_section(); + f->dump_string("name", name); + f->dump_unsigned("protection_status", protection_status); +} + +std::ostream& operator<<(std::ostream& os, const SnapState& snap_state) { + os << "[" + << "namespace=" << snap_state.snap_namespace << ", " + << "name=" << snap_state.name << ", " + << "protection=" << static_cast(snap_state.protection_status) + << "]"; + return os; +} + +void ImageState::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(name, bl); + encode(features, bl); + encode(snap_limit, bl); + encode(snapshots, bl); + encode(metadata, bl); + ENCODE_FINISH(bl); +} + +void ImageState::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(name, bl); + decode(features, bl); + decode(snap_limit, bl); + decode(snapshots, bl); + decode(metadata, bl); + DECODE_FINISH(bl); +} + +void ImageState::dump(Formatter *f) const { + f->dump_string("name", name); + f->dump_unsigned("features", features); + f->dump_unsigned("snap_limit", snap_limit); + f->open_array_section("snapshots"); + for (auto &[id, snap_state] : snapshots) { + f->open_object_section(stringify(id).c_str()); + snap_state.dump(f); + f->close_section(); // snap_state + } + f->close_section(); // snapshots + f->open_object_section("metadata"); + for (auto &it : metadata) { + f->dump_stream(it.first.c_str()) << it.second; + } + f->close_section(); // metadata +} + +std::ostream& operator<<(std::ostream& os, const ImageState& image_state) { + os << "[" + << "name=" << image_state.name << ", " + << "features=" << image_state.features << ", " + << "snap_limit=" << image_state.snap_limit << ", " + << "snaps=" << image_state.snapshots << ", " + << "metadata_count=" << image_state.metadata.size() + << "]"; + return os; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd diff --git a/src/librbd/mirror/snapshot/Types.h b/src/librbd/mirror/snapshot/Types.h new file mode 100644 index 000000000..79947a5f8 --- /dev/null +++ b/src/librbd/mirror/snapshot/Types.h @@ -0,0 +1,122 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H + +#include "cls/rbd/cls_rbd_types.h" +#include "include/buffer.h" +#include "include/types.h" + +#include +#include + +namespace librbd { +namespace mirror { +namespace snapshot { + +enum CreatePrimaryFlags { + CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS = (1 << 0), + CREATE_PRIMARY_FLAG_DEMOTED = (1 << 1), + CREATE_PRIMARY_FLAG_FORCE = (1 << 2) +}; + +struct ImageStateHeader { + uint32_t object_count = 0; + + ImageStateHeader() { + } + ImageStateHeader(uint32_t object_count) : object_count(object_count) { + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); +}; + +WRITE_CLASS_ENCODER(ImageStateHeader); + +struct SnapState { + cls::rbd::SnapshotNamespace snap_namespace; + std::string name; + uint8_t protection_status = 0; + + SnapState() { + } + SnapState(const cls::rbd::SnapshotNamespace &snap_namespace, + const std::string &name, uint8_t protection_status) + : snap_namespace(snap_namespace), name(name), + protection_status(protection_status) { + } + + bool operator==(const SnapState& rhs) const { + return snap_namespace == rhs.snap_namespace && + name == rhs.name && protection_status == rhs.protection_status; + } + + bool operator<(const SnapState& rhs) const { + if (snap_namespace != rhs.snap_namespace) { + return snap_namespace < rhs.snap_namespace; + } + if (name != rhs.name) { + return name < rhs.name; + } + return protection_status < rhs.protection_status; + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); + void dump(Formatter *f) const; +}; + +std::ostream& operator<<(std::ostream& os, const SnapState& snap_state); + +WRITE_CLASS_ENCODER(SnapState); + +struct ImageState { + std::string name; + uint64_t features = 0; + uint64_t snap_limit = 0; + std::map snapshots; + std::map metadata; + + ImageState() { + } + ImageState(const std::string &name, uint64_t features, uint64_t snap_limit, + const std::map &snapshots, + const std::map &metadata) + : name(name), features(features), snap_limit(snap_limit), + snapshots(snapshots), metadata(metadata) { + } + + bool operator==(const ImageState& rhs) const { + return name == rhs.name && features == rhs.features && + snap_limit == rhs.snap_limit && snapshots == rhs.snapshots; + } + + bool operator<(const ImageState& rhs) const { + if (name != rhs.name) { + return name < rhs.name; + } + if (features != rhs.features) { + return features < rhs.features; + } + if (snap_limit != rhs.snap_limit) { + return snap_limit < rhs.snap_limit; + } + return snapshots < rhs.snapshots; + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); + void dump(Formatter *f) const; +}; + +std::ostream& operator<<(std::ostream& os, const ImageState& image_state); + +WRITE_CLASS_ENCODER(ImageState); + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc new file mode 100644 index 000000000..35313f627 --- /dev/null +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc @@ -0,0 +1,230 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::UnlinkPeerRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void UnlinkPeerRequest::send() { + if (!m_image_ctx->state->is_refresh_required()) { + unlink_peer(); + return; + } + + refresh_image(); +} + +template +void UnlinkPeerRequest::refresh_image() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template +void UnlinkPeerRequest::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + unlink_peer(); +} + +template +void UnlinkPeerRequest::unlink_peer() { + CephContext *cct = m_image_ctx->cct; + + m_image_ctx->image_lock.lock_shared(); + int r = -ENOENT; + cls::rbd::SnapshotNamespace snap_namespace; + std::string snap_name; + bool have_newer_mirror_snapshot = false; + for (auto snap_it = m_image_ctx->snap_info.find(m_snap_id); + snap_it != m_image_ctx->snap_info.end(); ++snap_it) { + if (snap_it->first == m_snap_id) { + r = 0; + snap_namespace = snap_it->second.snap_namespace; + snap_name = snap_it->second.name; + } else if (std::holds_alternative( + snap_it->second.snap_namespace)) { + ldout(cct, 15) << "located newer mirror snapshot" << dendl; + have_newer_mirror_snapshot = true; + break; + } + } + + if (r == -ENOENT) { + ldout(cct, 15) << "missing snapshot: snap_id=" << m_snap_id << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(r); + return; + } + + auto mirror_ns = std::get_if( + &snap_namespace); + if (mirror_ns == nullptr) { + lderr(cct) << "not mirror snapshot (snap_id=" << m_snap_id << ")" << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(-EINVAL); + return; + } + + // if there is or will be no more peers in the mirror snapshot and we have + // a more recent mirror snapshot, remove the older one + if ((mirror_ns->mirror_peer_uuids.empty() || + (mirror_ns->mirror_peer_uuids.size() == 1 && + mirror_ns->mirror_peer_uuids.count(m_mirror_peer_uuid) != 0)) && + have_newer_mirror_snapshot) { + if (m_allow_remove) { + m_image_ctx->image_lock.unlock_shared(); + remove_snapshot(snap_namespace, snap_name); + return; + } else { + ldout(cct, 15) << "skipping removal of snapshot: snap_id=" << m_snap_id + << ", mirror_peer_uuid=" << m_mirror_peer_uuid + << ", mirror_peer_uuids=" << mirror_ns->mirror_peer_uuids + << dendl; + } + } + + if (mirror_ns->mirror_peer_uuids.count(m_mirror_peer_uuid) == 0) { + ldout(cct, 15) << "no peer to unlink: snap_id=" << m_snap_id + << ", mirror_peer_uuid=" << m_mirror_peer_uuid + << ", mirror_peer_uuids=" << mirror_ns->mirror_peer_uuids + << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(0); + return; + } + + m_image_ctx->image_lock.unlock_shared(); + + ldout(cct, 15) << "snap_id=" << m_snap_id << ", " + << "mirror_peer_uuid=" << m_mirror_peer_uuid << dendl; + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_snapshot_unlink_peer(&op, m_snap_id, + m_mirror_peer_uuid); + auto aio_comp = create_rados_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_unlink_peer>(this); + r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void UnlinkPeerRequest::handle_unlink_peer(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == -ERESTART || r == -ENOENT) { + if (r == -ERESTART) { + ldout(cct, 15) << "unlinking last peer not supported" << dendl; + m_allow_remove = true; + } + refresh_image(); + return; + } + + if (r < 0) { + lderr(cct) << "failed to unlink peer: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + notify_update(); +} + +template +void UnlinkPeerRequest::notify_update() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_notify_update>(this); + m_image_ctx->notify_update(ctx); +} + +template +void UnlinkPeerRequest::handle_notify_update(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == -ENOENT || r == -ETIMEDOUT) { + // non-fatel errors + lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; + } else if (r < 0) { + lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + refresh_image(); +} + +template +void UnlinkPeerRequest::remove_snapshot( + const cls::rbd::SnapshotNamespace& snap_namespace, + const std::string& snap_name) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_remove_snapshot>(this); + m_image_ctx->operations->snap_remove(snap_namespace, snap_name, ctx); +} + +template +void UnlinkPeerRequest::handle_remove_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove snapshot: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void UnlinkPeerRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + auto on_finish = m_on_finish; + delete this; + on_finish->complete(r); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::UnlinkPeerRequest; diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.h b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h new file mode 100644 index 000000000..192b40d6e --- /dev/null +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h @@ -0,0 +1,98 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H + +#include "include/buffer.h" +#include "cls/rbd/cls_rbd_client.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class UnlinkPeerRequest { +public: + static UnlinkPeerRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + const std::string &mirror_peer_uuid, + bool allow_remove, Context *on_finish) { + return new UnlinkPeerRequest(image_ctx, snap_id, mirror_peer_uuid, + allow_remove, on_finish); + } + + UnlinkPeerRequest(ImageCtxT *image_ctx, uint64_t snap_id, + const std::string &mirror_peer_uuid, bool allow_remove, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), + m_mirror_peer_uuid(mirror_peer_uuid), m_allow_remove(allow_remove), + m_on_finish(on_finish) { + } + + void send(); + +private: + /* + * @verbatim + * + * + * | + * v + * REFRESH_IMAGE <--------------------------\ + * | ^ (not found | + * | * or last) | + * | * | + * |\---------------> UNLINK_PEER --> NOTIFY_UPDATE + * | (not last peer or + * | no newer mirror + * | snap exists) + * | + * |\---------------> REMOVE_SNAPSHOT + * | (last peer and | + * | newer mirror | + * | snap exists) | + * | | + * |(peer not found) | + * v | + * <---------------/ + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + std::string m_mirror_peer_uuid; + bool m_allow_remove; + Context *m_on_finish; + + void refresh_image(); + void handle_refresh_image(int r); + + void unlink_peer(); + void handle_unlink_peer(int r); + + void notify_update(); + void handle_notify_update(int r); + + void remove_snapshot(const cls::rbd::SnapshotNamespace& snap_namespace, + const std::string& snap_name); + void handle_remove_snapshot(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::UnlinkPeerRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H diff --git a/src/librbd/mirror/snapshot/Utils.cc b/src/librbd/mirror/snapshot/Utils.cc new file mode 100644 index 000000000..36d1558be --- /dev/null +++ b/src/librbd/mirror/snapshot/Utils.cc @@ -0,0 +1,186 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/dout.h" +#include "common/errno.h" +#include "include/stringify.h" +#include "librbd/ImageCtx.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::util: " \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { +namespace util { + +namespace { + +const std::string IMAGE_STATE_OBJECT_PREFIX = "rbd_mirror_snapshot."; + +bool get_rollback_snap_id( + std::map::reverse_iterator it, + std::map::reverse_iterator end, + uint64_t *rollback_snap_id) { + + for (; it != end; it++) { + auto mirror_ns = std::get( + it->second.snap_namespace); + if (mirror_ns.state != cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + break; + } + if (mirror_ns.complete) { + break; + } + } + + if (it != end) { + *rollback_snap_id = it->first; + return true; + } + + return false; +} + +} // anonymous namespace + +std::string get_image_meta_key(const std::string& mirror_uuid) { + return ".rbd_mirror." + mirror_uuid; +} + +template +bool can_create_primary_snapshot(I *image_ctx, bool demoted, bool force, + bool* requires_orphan, + uint64_t *rollback_snap_id) { + CephContext *cct = image_ctx->cct; + + if (requires_orphan != nullptr) { + *requires_orphan = false; + } + if (rollback_snap_id) { + *rollback_snap_id = CEPH_NOSNAP; + } + + std::shared_lock image_locker{image_ctx->image_lock}; + + for (auto it = image_ctx->snap_info.rbegin(); + it != image_ctx->snap_info.rend(); it++) { + auto mirror_ns = std::get_if( + &it->second.snap_namespace); + if (mirror_ns == nullptr) { + continue; + } + ldout(cct, 20) << "previous snapshot snap_id=" << it->first << " " + << *mirror_ns << dendl; + if (mirror_ns->is_demoted() && !force) { + lderr(cct) << "trying to create primary snapshot without force " + << "when previous primary snapshot is demoted" + << dendl; + return false; + } + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + if (!force) { + lderr(cct) << "trying to create primary snapshot without force " + << "when previous snapshot is non-primary" + << dendl; + return false; + } + if (demoted) { + lderr(cct) << "trying to create primary demoted snapshot " + << "when previous snapshot is non-primary" + << dendl; + return false; + } + + if (requires_orphan != nullptr) { + *requires_orphan = !mirror_ns->is_demoted(); + } + if (!mirror_ns->complete) { + ldout(cct, 20) << "needs rollback" << dendl; + if (!rollback_snap_id) { + lderr(cct) << "trying to create primary snapshot " + << "when previous non-primary snapshot is not copied yet" + << dendl; + return false; + } + if (!get_rollback_snap_id(++it, image_ctx->snap_info.rend(), + rollback_snap_id)) { + lderr(cct) << "cannot rollback" << dendl; + return false; + } + ldout(cct, 20) << "rollback_snap_id=" << *rollback_snap_id << dendl; + } + return true; + } + + return true; + } + + ldout(cct, 20) << "no previous mirror snapshots found" << dendl; + return true; +} + +template +bool can_create_non_primary_snapshot(I *image_ctx) { + CephContext *cct = image_ctx->cct; + + std::shared_lock image_locker{image_ctx->image_lock}; + + for (auto it = image_ctx->snap_info.rbegin(); + it != image_ctx->snap_info.rend(); it++) { + auto mirror_ns = std::get_if( + &it->second.snap_namespace); + if (mirror_ns != nullptr) { + ldout(cct, 20) << "previous mirror snapshot snap_id=" << it->first << " " + << *mirror_ns << dendl; + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + if (!mirror_ns->complete) { + lderr(cct) << "trying to create non-primary snapshot " + << "when previous non-primary snapshot is not copied yet" + << dendl; + return false; + } + return true; + } + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(cct) << "trying to create non-primary snapshot " + << "when previous primary snapshot is not in demoted state" + << dendl; + return false; + } + return true; + } + } + + ldout(cct, 20) << "no previous mirror snapshots found" << dendl; + return true; +} + +template +std::string image_state_object_name(I *image_ctx, uint64_t snap_id, + uint64_t index) { + return IMAGE_STATE_OBJECT_PREFIX + image_ctx->id + "." + + stringify(snap_id) + "." + stringify(index); +} + +} // namespace util +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template bool librbd::mirror::snapshot::util::can_create_primary_snapshot( + librbd::ImageCtx *image_ctx, bool demoted, bool force, + bool* requires_orphan, uint64_t *rollback_snap_id); + +template bool librbd::mirror::snapshot::util::can_create_non_primary_snapshot( + librbd::ImageCtx *image_ctx); + +template std::string librbd::mirror::snapshot::util::image_state_object_name( + librbd::ImageCtx *image_ctx, uint64_t snap_id, uint64_t index); diff --git a/src/librbd/mirror/snapshot/Utils.h b/src/librbd/mirror/snapshot/Utils.h new file mode 100644 index 000000000..127ec5865 --- /dev/null +++ b/src/librbd/mirror/snapshot/Utils.h @@ -0,0 +1,38 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H + +#include "include/int_types.h" +#include "include/stringify.h" +#include + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { +namespace util { + +std::string get_image_meta_key(const std::string& mirror_uuid); + +template +bool can_create_primary_snapshot(ImageCtxT *image_ctx, bool demoted, bool force, + bool* requires_orphan, + uint64_t *rollback_snap_id); + +template +bool can_create_non_primary_snapshot(ImageCtxT *image_ctx); + +template +std::string image_state_object_name(ImageCtxT *image_ctx, uint64_t snap_id, + uint64_t index); + +} // namespace util +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H diff --git a/src/librbd/mirror/snapshot/WriteImageStateRequest.cc b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc new file mode 100644 index 000000000..c79dd7e2c --- /dev/null +++ b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc @@ -0,0 +1,120 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::WriteImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +namespace { + +static size_t header_length() { + bufferlist bl; + ImageStateHeader header; + + using ceph::encode; + encode(header, bl); + + return bl.length(); +} + +} +using librbd::util::create_rados_callback; + +template +WriteImageStateRequest::WriteImageStateRequest(I *image_ctx, + uint64_t snap_id, + const ImageState &image_state, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_image_state(image_state), + m_on_finish(on_finish), m_object_size( + 1 << image_ctx->config.template get_val("rbd_default_order")) { + bufferlist bl; + encode(m_image_state, bl); + + m_object_count = 1 + (header_length() + bl.length()) / m_object_size; + ImageStateHeader header(m_object_count); + + encode(header, m_bl); + m_bl.claim_append(bl); +} + +template +void WriteImageStateRequest::send() { + write_object(); +} + +template +void WriteImageStateRequest::write_object() { + CephContext *cct = m_image_ctx->cct; + ceph_assert(m_object_count > 0); + + m_object_count--; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_count); + ldout(cct, 15) << oid << dendl; + + size_t off = m_object_count * m_object_size; + size_t len = std::min(m_bl.length() - off, m_object_size); + bufferlist bl; + bl.substr_of(m_bl, off, len); + + librados::ObjectWriteOperation op; + op.write_full(bl); + + librados::AioCompletion *comp = create_rados_callback< + WriteImageStateRequest, + &WriteImageStateRequest::handle_write_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void WriteImageStateRequest::handle_write_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_object_count == 0) { + finish(0); + return; + } + + write_object(); +} + +template +void WriteImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::WriteImageStateRequest; diff --git a/src/librbd/mirror/snapshot/WriteImageStateRequest.h b/src/librbd/mirror/snapshot/WriteImageStateRequest.h new file mode 100644 index 000000000..d2c4a7f80 --- /dev/null +++ b/src/librbd/mirror/snapshot/WriteImageStateRequest.h @@ -0,0 +1,73 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H + +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class WriteImageStateRequest { +public: + static WriteImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + const ImageState &image_state, + Context *on_finish) { + return new WriteImageStateRequest(image_ctx, snap_id, image_state, + on_finish); + } + + WriteImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + const ImageState &image_state, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * WRITE_OBJECT (repeat for + * | every object) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + ImageState m_image_state; + Context *m_on_finish; + + bufferlist m_bl; + + const size_t m_object_size; + size_t m_object_count = 0; + + void write_object(); + void handle_write_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::WriteImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H -- cgit v1.2.3