From e6918187568dbd01842d8d1d2c808ce16a894239 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:54:28 +0200 Subject: Adding upstream version 18.2.2. Signed-off-by: Daniel Baumann --- .../mirror/snapshot/CreateNonPrimaryRequest.cc | 273 ++++++++++++++ .../mirror/snapshot/CreateNonPrimaryRequest.h | 123 +++++++ src/librbd/mirror/snapshot/CreatePrimaryRequest.cc | 281 ++++++++++++++ src/librbd/mirror/snapshot/CreatePrimaryRequest.h | 106 ++++++ src/librbd/mirror/snapshot/DemoteRequest.cc | 110 ++++++ src/librbd/mirror/snapshot/DemoteRequest.h | 76 ++++ src/librbd/mirror/snapshot/GetImageStateRequest.cc | 114 ++++++ src/librbd/mirror/snapshot/GetImageStateRequest.h | 76 ++++ src/librbd/mirror/snapshot/ImageMeta.cc | 175 +++++++++ src/librbd/mirror/snapshot/ImageMeta.h | 78 ++++ src/librbd/mirror/snapshot/PromoteRequest.cc | 405 +++++++++++++++++++++ src/librbd/mirror/snapshot/PromoteRequest.h | 151 ++++++++ .../mirror/snapshot/RemoveImageStateRequest.cc | 131 +++++++ .../mirror/snapshot/RemoveImageStateRequest.h | 75 ++++ src/librbd/mirror/snapshot/SetImageStateRequest.cc | 235 ++++++++++++ src/librbd/mirror/snapshot/SetImageStateRequest.h | 96 +++++ src/librbd/mirror/snapshot/Types.cc | 109 ++++++ src/librbd/mirror/snapshot/Types.h | 122 +++++++ src/librbd/mirror/snapshot/UnlinkPeerRequest.cc | 230 ++++++++++++ src/librbd/mirror/snapshot/UnlinkPeerRequest.h | 98 +++++ src/librbd/mirror/snapshot/Utils.cc | 186 ++++++++++ src/librbd/mirror/snapshot/Utils.h | 38 ++ .../mirror/snapshot/WriteImageStateRequest.cc | 120 ++++++ .../mirror/snapshot/WriteImageStateRequest.h | 73 ++++ 24 files changed, 3481 insertions(+) create mode 100644 src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc create mode 100644 src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h create mode 100644 src/librbd/mirror/snapshot/CreatePrimaryRequest.cc create mode 100644 src/librbd/mirror/snapshot/CreatePrimaryRequest.h create mode 100644 src/librbd/mirror/snapshot/DemoteRequest.cc create mode 100644 src/librbd/mirror/snapshot/DemoteRequest.h create mode 100644 src/librbd/mirror/snapshot/GetImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/GetImageStateRequest.h create mode 100644 src/librbd/mirror/snapshot/ImageMeta.cc create mode 100644 src/librbd/mirror/snapshot/ImageMeta.h create mode 100644 src/librbd/mirror/snapshot/PromoteRequest.cc create mode 100644 src/librbd/mirror/snapshot/PromoteRequest.h create mode 100644 src/librbd/mirror/snapshot/RemoveImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/RemoveImageStateRequest.h create mode 100644 src/librbd/mirror/snapshot/SetImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/SetImageStateRequest.h create mode 100644 src/librbd/mirror/snapshot/Types.cc create mode 100644 src/librbd/mirror/snapshot/Types.h create mode 100644 src/librbd/mirror/snapshot/UnlinkPeerRequest.cc create mode 100644 src/librbd/mirror/snapshot/UnlinkPeerRequest.h create mode 100644 src/librbd/mirror/snapshot/Utils.cc create mode 100644 src/librbd/mirror/snapshot/Utils.h create mode 100644 src/librbd/mirror/snapshot/WriteImageStateRequest.cc create mode 100644 src/librbd/mirror/snapshot/WriteImageStateRequest.h (limited to 'src/librbd/mirror/snapshot') diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc new file mode 100644 index 000000000..eed0aa506 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc @@ -0,0 +1,273 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Utils.h" +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::CreateNonPrimaryRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +CreateNonPrimaryRequest::CreateNonPrimaryRequest( + I* image_ctx, bool demoted, const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, const SnapSeqs& snap_seqs, + const ImageState &image_state, uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_demoted(demoted), + m_primary_mirror_uuid(primary_mirror_uuid), + m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs), + m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) { + m_default_ns_ctx.dup(m_image_ctx->md_ctx); + m_default_ns_ctx.set_namespace(""); +} + +template +void CreateNonPrimaryRequest::send() { + refresh_image(); +} + +template +void CreateNonPrimaryRequest::refresh_image() { + if (!m_image_ctx->state->is_refresh_required()) { + get_mirror_image(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template +void CreateNonPrimaryRequest::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_mirror_image(); +} + +template +void CreateNonPrimaryRequest::get_mirror_image() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_ctx->id); + + librados::AioCompletion *comp = create_rados_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_get_mirror_image>(this); + int r = m_image_ctx->md_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void CreateNonPrimaryRequest::handle_get_mirror_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + cls::rbd::MirrorImage mirror_image; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_image_get_finish(&iter, &mirror_image); + } + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + lderr(cct) << "snapshot based mirroring is not enabled" << dendl; + finish(-EINVAL); + return; + } + + if (!is_orphan() && !util::can_create_non_primary_snapshot(m_image_ctx)) { + finish(-EINVAL); + return; + } + + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_snap_name = ".mirror.non_primary." + mirror_image.global_image_id + "." + + uuid_gen.to_string(); + + get_mirror_peers(); +} + +template +void CreateNonPrimaryRequest::get_mirror_peers() { + if (!m_demoted) { + create_snapshot(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + auto aio_comp = create_rados_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_get_mirror_peers>(this); + m_out_bl.clear(); + int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void CreateNonPrimaryRequest::handle_get_mirror_peers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + std::vector peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + + create_snapshot(); +} + +template +void CreateNonPrimaryRequest::create_snapshot() { + CephContext *cct = m_image_ctx->cct; + + cls::rbd::MirrorSnapshotNamespace ns{ + (m_demoted ? cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED : + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY), {}, + m_primary_mirror_uuid, m_primary_snap_id}; + if (m_demoted) { + ns.mirror_peer_uuids = m_mirror_peer_uuids; + } + ns.snap_seqs = m_snap_seqs; + ns.complete = is_orphan(); + ldout(cct, 15) << "ns=" << ns << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_create_snapshot>(this); + m_image_ctx->operations->snap_create(ns, m_snap_name, 0, m_prog_ctx, ctx); +} + +template +void CreateNonPrimaryRequest::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + write_image_state(); +} + +template +void CreateNonPrimaryRequest::write_image_state() { + uint64_t snap_id; + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + snap_id = m_image_ctx->get_snap_id( + cls::rbd::MirrorSnapshotNamespace{}, m_snap_name); + } + + if (m_snap_id != nullptr) { + *m_snap_id = snap_id; + } + + if (is_orphan()) { + finish(0); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest, + &CreateNonPrimaryRequest::handle_write_image_state>(this); + + auto req = WriteImageStateRequest::create(m_image_ctx, snap_id, + m_image_state, ctx); + req->send(); +} + +template +void CreateNonPrimaryRequest::handle_write_image_state(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write image state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void CreateNonPrimaryRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::CreateNonPrimaryRequest; diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h new file mode 100644 index 000000000..36f155413 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h @@ -0,0 +1,123 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H + +#include "include/buffer.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/Types.h" +#include "librbd/internal.h" +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class CreateNonPrimaryRequest { +public: + static CreateNonPrimaryRequest *create(ImageCtxT *image_ctx, + bool demoted, + const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, + const SnapSeqs& snap_seqs, + const ImageState &image_state, + uint64_t *snap_id, + Context *on_finish) { + return new CreateNonPrimaryRequest(image_ctx, demoted, primary_mirror_uuid, + primary_snap_id, snap_seqs, image_state, + snap_id, on_finish); + } + + CreateNonPrimaryRequest(ImageCtxT *image_ctx, + bool demoted, + const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, + const SnapSeqs& snap_seqs, + const ImageState &image_state, uint64_t *snap_id, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * REFRESH_IMAGE + * | + * v + * GET_MIRROR_IMAGE + * | + * v (skip if not needed) + * GET_MIRROR_PEERS + * | + * v + * CREATE_SNAPSHOT + * | + * v + * WRITE_IMAGE_STATE + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + bool m_demoted; + std::string m_primary_mirror_uuid; + uint64_t m_primary_snap_id; + SnapSeqs m_snap_seqs; + ImageState m_image_state; + uint64_t *m_snap_id; + Context *m_on_finish; + + librados::IoCtx m_default_ns_ctx; + std::set m_mirror_peer_uuids; + + std::string m_snap_name; + + bufferlist m_out_bl; + NoOpProgressContext m_prog_ctx; + + bool is_orphan() const { + return m_primary_mirror_uuid.empty(); + } + + void refresh_image(); + void handle_refresh_image(int r); + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void get_mirror_peers(); + void handle_get_mirror_peers(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void write_image_state(); + void handle_write_image_state(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::CreateNonPrimaryRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc new file mode 100644 index 000000000..c8e3a4fe7 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc @@ -0,0 +1,281 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::CreatePrimaryRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +CreatePrimaryRequest::CreatePrimaryRequest( + I *image_ctx, const std::string& global_image_id, + uint64_t clean_since_snap_id, uint64_t snap_create_flags, uint32_t flags, + uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_clean_since_snap_id(clean_since_snap_id), + m_snap_create_flags(snap_create_flags), m_flags(flags), m_snap_id(snap_id), + m_on_finish(on_finish) { + m_default_ns_ctx.dup(m_image_ctx->md_ctx); + m_default_ns_ctx.set_namespace(""); +} + +template +void CreatePrimaryRequest::send() { + if (!util::can_create_primary_snapshot( + m_image_ctx, + ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0), + ((m_flags & CREATE_PRIMARY_FLAG_FORCE) != 0), nullptr, nullptr)) { + finish(-EINVAL); + return; + } + + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_snap_name = ".mirror.primary." + m_global_image_id + "." + + uuid_gen.to_string(); + + get_mirror_peers(); +} + +template +void CreatePrimaryRequest::get_mirror_peers() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + librados::AioCompletion *comp = create_rados_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_get_mirror_peers>(this); + m_out_bl.clear(); + int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void CreatePrimaryRequest::handle_get_mirror_peers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + std::vector peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + + if (m_mirror_peer_uuids.empty() && + ((m_flags & CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS) == 0)) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + finish(-EINVAL); + return; + } + + create_snapshot(); +} + +template +void CreatePrimaryRequest::create_snapshot() { + cls::rbd::MirrorSnapshotNamespace ns{ + ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0 ? + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED : + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY), + m_mirror_peer_uuids, "", m_clean_since_snap_id}; + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "name=" << m_snap_name << ", " + << "ns=" << ns << dendl; + auto ctx = create_context_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_create_snapshot>(this); + m_image_ctx->operations->snap_create(ns, m_snap_name, m_snap_create_flags, + m_prog_ctx, ctx); +} + +template +void CreatePrimaryRequest::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + refresh_image(); +} + +template +void CreatePrimaryRequest::refresh_image() { + // refresh is required to retrieve the snapshot id (if snapshot + // created via remote RPC) and complete flag (regardless) + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template +void CreatePrimaryRequest::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + if (m_snap_id != nullptr) { + std::shared_lock image_locker{m_image_ctx->image_lock}; + *m_snap_id = m_image_ctx->get_snap_id( + cls::rbd::MirrorSnapshotNamespace{}, m_snap_name); + ldout(cct, 15) << "snap_id=" << *m_snap_id << dendl; + } + + unlink_peer(); +} + +template +void CreatePrimaryRequest::unlink_peer() { + // TODO: Document semantics for unlink_peer + uint64_t max_snapshots = m_image_ctx->config.template get_val( + "rbd_mirroring_max_mirroring_snapshots"); + ceph_assert(max_snapshots >= 3); + + std::string peer_uuid; + uint64_t snap_id = CEPH_NOSNAP; + + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + for (const auto& peer : m_mirror_peer_uuids) { + for (const auto& snap_info_pair : m_image_ctx->snap_info) { + auto info = std::get_if( + &snap_info_pair.second.snap_namespace); + if (info == nullptr) { + continue; + } + if (info->mirror_peer_uuids.empty() || + (info->mirror_peer_uuids.count(peer) != 0 && + info->is_primary() && !info->complete)) { + peer_uuid = peer; + snap_id = snap_info_pair.first; + goto do_unlink; + } + } + } + for (const auto& peer : m_mirror_peer_uuids) { + size_t count = 0; + uint64_t unlink_snap_id = 0; + for (const auto& snap_info_pair : m_image_ctx->snap_info) { + auto info = std::get_if( + &snap_info_pair.second.snap_namespace); + if (info == nullptr) { + continue; + } + if (info->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + // reset counters -- we count primary snapshots after the last + // promotion + count = 0; + unlink_snap_id = 0; + continue; + } + if (info->mirror_peer_uuids.count(peer) == 0) { + // snapshot is not linked with this peer + continue; + } + count++; + if (count == max_snapshots) { + unlink_snap_id = snap_info_pair.first; + } + if (count > max_snapshots) { + peer_uuid = peer; + snap_id = unlink_snap_id; + goto do_unlink; + } + } + } + } + + finish(0); + return; + +do_unlink: + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "peer=" << peer_uuid << ", snap_id=" << snap_id << dendl; + + auto ctx = create_context_callback< + CreatePrimaryRequest, + &CreatePrimaryRequest::handle_unlink_peer>(this); + auto req = UnlinkPeerRequest::create(m_image_ctx, snap_id, peer_uuid, true, + ctx); + req->send(); +} + +template +void CreatePrimaryRequest::handle_unlink_peer(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to unlink peer: " << cpp_strerror(r) << dendl; + finish(0); // not fatal + return; + } + + unlink_peer(); +} + +template +void CreatePrimaryRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::CreatePrimaryRequest; diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.h b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h new file mode 100644 index 000000000..b8e84cf2b --- /dev/null +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h @@ -0,0 +1,106 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/internal.h" +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class CreatePrimaryRequest { +public: + static CreatePrimaryRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + uint64_t clean_since_snap_id, + uint64_t snap_create_flags, + uint32_t flags, uint64_t *snap_id, + Context *on_finish) { + return new CreatePrimaryRequest(image_ctx, global_image_id, + clean_since_snap_id, snap_create_flags, flags, + snap_id, on_finish); + } + + CreatePrimaryRequest(ImageCtxT *image_ctx, + const std::string& global_image_id, + uint64_t clean_since_snap_id, uint64_t snap_create_flags, + uint32_t flags, uint64_t *snap_id, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_MIRROR_PEERS + * | + * v + * CREATE_SNAPSHOT + * | + * v + * REFRESH_IMAGE + * | + * v + * UNLINK_PEER (skip if not needed, + * | repeat if needed) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + uint64_t m_clean_since_snap_id; + const uint64_t m_snap_create_flags; + const uint32_t m_flags; + uint64_t *m_snap_id; + Context *m_on_finish; + + librados::IoCtx m_default_ns_ctx; + std::set m_mirror_peer_uuids; + std::string m_snap_name; + + bufferlist m_out_bl; + NoOpProgressContext m_prog_ctx; + + void get_mirror_peers(); + void handle_get_mirror_peers(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void refresh_image(); + void handle_refresh_image(int r); + + void unlink_peer(); + void handle_unlink_peer(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::CreatePrimaryRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H diff --git a/src/librbd/mirror/snapshot/DemoteRequest.cc b/src/librbd/mirror/snapshot/DemoteRequest.cc new file mode 100644 index 000000000..ccaa33c83 --- /dev/null +++ b/src/librbd/mirror/snapshot/DemoteRequest.cc @@ -0,0 +1,110 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/DemoteRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::DemoteRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void DemoteRequest::send() { + enable_non_primary_feature(); +} + +template +void DemoteRequest::enable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // ensure image is flagged with non-primary feature so that + // standard RBD clients cannot write to it. + librados::ObjectWriteOperation op; + cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY, + RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + DemoteRequest, + &DemoteRequest::handle_enable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void DemoteRequest::handle_enable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to enable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_snapshot(); +} + +template +void DemoteRequest::create_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + DemoteRequest, &DemoteRequest::handle_create_snapshot>(this); + + auto req = CreatePrimaryRequest::create( + m_image_ctx, m_global_image_id, CEPH_NOSNAP, + SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, + (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | + snapshot::CREATE_PRIMARY_FLAG_DEMOTED), nullptr, ctx); + req->send(); +} + +template +void DemoteRequest::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void DemoteRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::DemoteRequest; diff --git a/src/librbd/mirror/snapshot/DemoteRequest.h b/src/librbd/mirror/snapshot/DemoteRequest.h new file mode 100644 index 000000000..63c935645 --- /dev/null +++ b/src/librbd/mirror/snapshot/DemoteRequest.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H + +#include "include/buffer.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class DemoteRequest { +public: + static DemoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + Context *on_finish) { + return new DemoteRequest(image_ctx, global_image_id, on_finish); + } + + DemoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * ENABLE_NON_PRIMARY_FEATURE + * | + * v + * CREATE_SNAPSHOT + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + Context *m_on_finish; + + void enable_non_primary_feature(); + void handle_enable_non_primary_feature(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void finish(int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::DemoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/GetImageStateRequest.cc b/src/librbd/mirror/snapshot/GetImageStateRequest.cc new file mode 100644 index 000000000..4692f88cb --- /dev/null +++ b/src/librbd/mirror/snapshot/GetImageStateRequest.cc @@ -0,0 +1,114 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/GetImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Types.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::GetImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + +template +void GetImageStateRequest::send() { + read_object(); +} + + +template +void GetImageStateRequest::read_object() { + CephContext *cct = m_image_ctx->cct; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_index); + ldout(cct, 15) << oid << dendl; + + librados::ObjectReadOperation op; + m_bl.clear(); + op.read(0, 0, &m_bl, nullptr); + + librados::AioCompletion *comp = create_rados_callback< + GetImageStateRequest, + &GetImageStateRequest::handle_read_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op, nullptr); + ceph_assert(r == 0); + comp->release(); +} + +template +void GetImageStateRequest::handle_read_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to read image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + auto iter = m_bl.cbegin(); + + if (m_object_index == 0) { + ImageStateHeader header; + try { + using ceph::decode; + decode(header, iter); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state object header" << dendl; + finish(-EBADMSG); + return; + } + m_object_count = header.object_count; + } + + bufferlist bl; + bl.substr_of(m_bl, iter.get_off(), m_bl.length() - iter.get_off()); + m_state_bl.claim_append(bl); + + m_object_index++; + + if (m_object_index >= m_object_count) { + finish(0); + return; + } + + read_object(); +} + +template +void GetImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + try { + using ceph::decode; + decode(*m_image_state, m_state_bl); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state" << dendl; + r = -EBADMSG; + } + } + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::GetImageStateRequest; diff --git a/src/librbd/mirror/snapshot/GetImageStateRequest.h b/src/librbd/mirror/snapshot/GetImageStateRequest.h new file mode 100644 index 000000000..483e3a228 --- /dev/null +++ b/src/librbd/mirror/snapshot/GetImageStateRequest.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H + +#include "include/buffer.h" +#include "include/types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +struct ImageState; + +template +class GetImageStateRequest { +public: + static GetImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + ImageState *image_state, + Context *on_finish) { + return new GetImageStateRequest(image_ctx, snap_id, image_state, on_finish); + } + + GetImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + ImageState *image_state, Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_image_state(image_state), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * READ_OBJECT (repeat for + * | every object) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + ImageState *m_image_state; + Context *m_on_finish; + + bufferlist m_bl; + bufferlist m_state_bl; + + size_t m_object_count = 0; + size_t m_object_index = 0; + + void read_object(); + void handle_read_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::GetImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/ImageMeta.cc b/src/librbd/mirror/snapshot/ImageMeta.cc new file mode 100644 index 000000000..826899775 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.cc @@ -0,0 +1,175 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "json_spirit/json_spirit.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/WatchNotifyTypes.h" +#include "librbd/mirror/snapshot/Utils.h" +#include "librbd/watcher/Notifier.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::ImageMeta: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; +using librbd::mirror::snapshot::util::get_image_meta_key; + +template +ImageMeta::ImageMeta(I* image_ctx, const std::string& mirror_uuid) + : m_image_ctx(image_ctx), m_mirror_uuid(mirror_uuid) { +} + +template +void ImageMeta::load(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + librados::ObjectReadOperation op; + cls_client::metadata_get_start(&op, get_image_meta_key(m_mirror_uuid)); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_load(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_load(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + std::string data; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = cls_client::metadata_get_finish(&it, &data); + } + + if (r == -ENOENT) { + ldout(m_image_ctx->cct, 15) << "no snapshot-based mirroring image-meta: " + << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } else if (r < 0) { + lderr(m_image_ctx->cct) << "failed to load snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + bool json_valid = false; + json_spirit::mValue json_root; + if (json_spirit::read(data, json_root)) { + try { + auto& json_obj = json_root.get_obj(); + resync_requested = json_obj["resync_requested"].get_bool(); + json_valid = true; + } catch (std::runtime_error&) { + } + } + + if (!json_valid) { + lderr(m_image_ctx->cct) << "invalid image-meta JSON received" << dendl; + on_finish->complete(-EBADMSG); + return; + } + + on_finish->complete(0); +} + +template +void ImageMeta::save(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + // simple implementation for now + std::string json = "{\"resync_requested\": " + + std::string(resync_requested ? "true" : "false") + "}"; + + bufferlist bl; + bl.append(json); + + // avoid using built-in metadata_set operation since that would require + // opening the non-primary image in read/write mode which isn't supported + librados::ObjectWriteOperation op; + cls_client::metadata_set(&op, {{get_image_meta_key(m_mirror_uuid), bl}}); + + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_save(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_save(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to save snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + notify_update(on_finish); +} + +template +void ImageMeta::notify_update(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << dendl; + + // directly send header notification on image since you cannot + // open a non-primary image read/write and therefore cannot re-use + // the ImageWatcher to send the notification + bufferlist bl; + encode(watch_notify::NotifyMessage(new watch_notify::HeaderUpdatePayload()), + bl); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_notify_update(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_notify( + m_image_ctx->header_oid, aio_comp, bl, watcher::Notifier::NOTIFY_TIMEOUT, + &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void ImageMeta::handle_notify_update(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to notify image update: " + << cpp_strerror(r) << dendl; + } + on_finish->complete(r); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::ImageMeta; diff --git a/src/librbd/mirror/snapshot/ImageMeta.h b/src/librbd/mirror/snapshot/ImageMeta.h new file mode 100644 index 000000000..5d05f1927 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.h @@ -0,0 +1,78 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H + +#include "include/rados/librados.hpp" +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class ImageMeta { +public: + static ImageMeta* create(ImageCtxT* image_ctx, + const std::string& mirror_uuid) { + return new ImageMeta(image_ctx, mirror_uuid); + } + + ImageMeta(ImageCtxT* image_ctx, const std::string& mirror_uuid); + + void load(Context* on_finish); + void save(Context* on_finish); + + bool resync_requested = false; + +private: + /** + * @verbatim + * + * + * | + * v + * METADATA_GET + * | + * v + * + * | + * v + * METADATA_SET + * | + * v + * NOTIFY_UPDATE + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT* m_image_ctx; + std::string m_mirror_uuid; + + bufferlist m_out_bl; + + void handle_load(Context* on_finish, int r); + + void handle_save(Context* on_finish, int r); + + void notify_update(Context* on_finish); + void handle_notify_update(Context* on_finish, int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::ImageMeta; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc new file mode 100644 index 000000000..9718c299e --- /dev/null +++ b/src/librbd/mirror/snapshot/PromoteRequest.cc @@ -0,0 +1,405 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/PromoteRequest.h" +#include "common/Timer.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/image/ListWatchersRequest.h" +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::PromoteRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void PromoteRequest::send() { + CephContext *cct = m_image_ctx->cct; + bool requires_orphan = false; + if (!util::can_create_primary_snapshot(m_image_ctx, false, true, + &requires_orphan, + &m_rollback_snap_id)) { + lderr(cct) << "cannot promote" << dendl; + finish(-EINVAL); + return; + } else if (m_rollback_snap_id == CEPH_NOSNAP && !requires_orphan) { + create_promote_snapshot(); + return; + } + + ldout(cct, 15) << "requires_orphan=" << requires_orphan << ", " + << "rollback_snap_id=" << m_rollback_snap_id << dendl; + create_orphan_snapshot(); +} + +template +void PromoteRequest::create_orphan_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_create_orphan_snapshot>(this); + + auto req = CreateNonPrimaryRequest::create( + m_image_ctx, false, "", CEPH_NOSNAP, {}, {}, nullptr, ctx); + req->send(); +} + +template +void PromoteRequest::handle_create_orphan_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create orphan snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + list_watchers(); +} + +template +void PromoteRequest::list_watchers() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_list_watchers>(this); + + m_watchers.clear(); + auto flags = librbd::image::LIST_WATCHERS_FILTER_OUT_MY_INSTANCE | + librbd::image::LIST_WATCHERS_MIRROR_INSTANCES_ONLY; + auto req = librbd::image::ListWatchersRequest::create( + *m_image_ctx, flags, &m_watchers, ctx); + req->send(); +} + +template +void PromoteRequest::handle_list_watchers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to list watchers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_watchers.empty()) { + acquire_exclusive_lock(); + return; + } + + wait_update_notify(); +} + +template +void PromoteRequest::wait_update_notify() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + ImageCtx::get_timer_instance(cct, &m_timer, &m_timer_lock); + + std::lock_guard timer_lock{*m_timer_lock}; + + m_scheduler_ticks = 5; + + int r = m_image_ctx->state->register_update_watcher(&m_update_watch_ctx, + &m_update_watcher_handle); + if (r < 0) { + lderr(cct) << "failed to register update watcher: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + scheduler_unregister_update_watcher(); +} + +template +void PromoteRequest::handle_update_notify() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + std::lock_guard timer_lock{*m_timer_lock}; + m_scheduler_ticks = 0; +} + +template +void PromoteRequest::scheduler_unregister_update_watcher() { + ceph_assert(ceph_mutex_is_locked(*m_timer_lock)); + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "scheduler_ticks=" << m_scheduler_ticks << dendl; + + if (m_scheduler_ticks > 0) { + m_scheduler_ticks--; + m_timer->add_event_after(1, new LambdaContext([this](int) { + scheduler_unregister_update_watcher(); + })); + return; + } + + m_image_ctx->op_work_queue->queue(new LambdaContext([this](int) { + unregister_update_watcher(); + }), 0); +} + +template +void PromoteRequest::unregister_update_watcher() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_unregister_update_watcher>(this); + + m_image_ctx->state->unregister_update_watcher(m_update_watcher_handle, ctx); +} + +template +void PromoteRequest::handle_unregister_update_watcher(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to unregister update watcher: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + list_watchers(); +} + +template +void PromoteRequest::acquire_exclusive_lock() { + { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr && + !m_image_ctx->exclusive_lock->is_lock_owner()) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + m_lock_acquired = true; + m_image_ctx->exclusive_lock->block_requests(0); + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_acquire_exclusive_lock>(this); + + m_image_ctx->exclusive_lock->acquire_lock(ctx); + return; + } + } + + rollback(); +} + +template +void PromoteRequest::handle_acquire_exclusive_lock(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to acquire exclusive lock: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr && + !m_image_ctx->exclusive_lock->is_lock_owner()) { + lderr(cct) << "failed to acquire exclusive lock" << dendl; + r = m_image_ctx->exclusive_lock->get_unlocked_op_error(); + locker.unlock(); + finish(r); + return; + } + } + + rollback(); +} + +template +void PromoteRequest::rollback() { + if (m_rollback_snap_id == CEPH_NOSNAP) { + create_promote_snapshot(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + std::shared_lock owner_locker{m_image_ctx->owner_lock}; + std::shared_lock image_locker{m_image_ctx->image_lock}; + + auto info = m_image_ctx->get_snap_info(m_rollback_snap_id); + ceph_assert(info != nullptr); + auto snap_namespace = info->snap_namespace; + auto snap_name = info->name; + + image_locker.unlock(); + + auto ctx = create_async_context_callback( + *m_image_ctx, create_context_callback< + PromoteRequest, &PromoteRequest::handle_rollback>(this)); + + m_image_ctx->operations->execute_snap_rollback(snap_namespace, snap_name, + m_progress_ctx, ctx); +} + +template +void PromoteRequest::handle_rollback(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to rollback: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_promote_snapshot(); +} + +template +void PromoteRequest::create_promote_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_create_promote_snapshot>(this); + + auto req = CreatePrimaryRequest::create( + m_image_ctx, m_global_image_id, CEPH_NOSNAP, + SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, + (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | + snapshot::CREATE_PRIMARY_FLAG_FORCE), nullptr, ctx); + req->send(); +} + +template +void PromoteRequest::handle_create_promote_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create promote snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + disable_non_primary_feature(); +} + +template +void PromoteRequest::disable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // remove the non-primary feature flag so that the image can be + // R/W by standard RBD clients + librados::ObjectWriteOperation op; + cls_client::set_features(&op, 0U, RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + PromoteRequest, + &PromoteRequest::handle_disable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void PromoteRequest::handle_disable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to disable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + release_exclusive_lock(); +} + +template +void PromoteRequest::release_exclusive_lock() { + if (m_lock_acquired) { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + m_image_ctx->exclusive_lock->unblock_requests(); + + auto ctx = create_context_callback< + PromoteRequest, + &PromoteRequest::handle_release_exclusive_lock>(this); + + m_image_ctx->exclusive_lock->release_lock(ctx); + return; + } + } + + finish(0); +} + +template +void PromoteRequest::handle_release_exclusive_lock(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to release exclusive lock: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void PromoteRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::PromoteRequest; diff --git a/src/librbd/mirror/snapshot/PromoteRequest.h b/src/librbd/mirror/snapshot/PromoteRequest.h new file mode 100644 index 000000000..1d9a862a0 --- /dev/null +++ b/src/librbd/mirror/snapshot/PromoteRequest.h @@ -0,0 +1,151 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H + +#include "include/buffer.h" +#include "include/rbd/librbd.hpp" +#include "common/ceph_mutex.h" +#include "common/Timer.h" +#include "librbd/internal.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class PromoteRequest { +public: + static PromoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + Context *on_finish) { + return new PromoteRequest(image_ctx, global_image_id, on_finish); + } + + PromoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * | (can promote) + * |\----------------------------------------\ + * | | + * | | + * v (skip if not needed) | + * CREATE_ORPHAN_SNAPSHOT | + * | | + * | /-- UNREGISTER_UPDATE_WATCHER <-\ | + * v v | | + * LIST_WATCHERS ----> WAIT_UPDATE_NOTIFY --/ | + * | | + * | (no watchers) | + * v | + * ACQUIRE_EXCLUSIVE_LOCK | + * | (skip if not needed) | + * v | + * ROLLBACK | + * | | + * v | + * CREATE_PROMOTE_SNAPSHOT <--------------------/ + * | + * v + * DISABLE_NON_PRIMARY_FEATURE + * | + * v + * RELEASE_EXCLUSIVE_LOCK (skip if not needed) + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + Context *m_on_finish; + + uint64_t m_rollback_snap_id = CEPH_NOSNAP; + bool m_lock_acquired = false; + NoOpProgressContext m_progress_ctx; + + class UpdateWatchCtx : public librbd::UpdateWatchCtx { + public: + UpdateWatchCtx(PromoteRequest *promote_request) + : promote_request(promote_request) { + } + + void handle_notify() { + promote_request->handle_update_notify(); + } + + private: + PromoteRequest *promote_request; + + } m_update_watch_ctx = {this}; + + std::list m_watchers; + uint64_t m_update_watcher_handle = 0; + uint64_t m_scheduler_ticks = 0; + SafeTimer *m_timer = nullptr; + ceph::mutex *m_timer_lock = nullptr; + + void refresh_image(); + void handle_refresh_image(int r); + + void create_orphan_snapshot(); + void handle_create_orphan_snapshot(int r); + + void list_watchers(); + void handle_list_watchers(int r); + + void wait_update_notify(); + void handle_update_notify(); + void scheduler_unregister_update_watcher(); + + void unregister_update_watcher(); + void handle_unregister_update_watcher(int r); + + void acquire_exclusive_lock(); + void handle_acquire_exclusive_lock(int r); + + void rollback(); + void handle_rollback(int r); + + void create_promote_snapshot(); + void handle_create_promote_snapshot(int r); + + void disable_non_primary_feature(); + void handle_disable_non_primary_feature(int r); + + void release_exclusive_lock(); + void handle_release_exclusive_lock(int r); + + void finish(int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::PromoteRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc new file mode 100644 index 000000000..204e0489a --- /dev/null +++ b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc @@ -0,0 +1,131 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/RemoveImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Types.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::RemoveImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + +template +void RemoveImageStateRequest::send() { + get_object_count(); +} + + +template +void RemoveImageStateRequest::get_object_count() { + CephContext *cct = m_image_ctx->cct; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, 0); + ldout(cct, 15) << oid << dendl; + + librados::ObjectReadOperation op; + op.read(0, 0, &m_bl, nullptr); + + librados::AioCompletion *comp = create_rados_callback< + RemoveImageStateRequest, + &RemoveImageStateRequest::handle_get_object_count>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op, nullptr); + ceph_assert(r == 0); + comp->release(); +} + +template +void RemoveImageStateRequest::handle_get_object_count(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to read image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ImageStateHeader header(1); + auto iter = m_bl.cbegin(); + try { + using ceph::decode; + + decode(header, iter); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state object header" << dendl; + // still try to remove it + } + + m_object_count = header.object_count > 0 ? header.object_count : 1; + + remove_object(); +} + +template +void RemoveImageStateRequest::remove_object() { + CephContext *cct = m_image_ctx->cct; + + ceph_assert(m_object_count > 0); + m_object_count--; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_count); + ldout(cct, 15) << oid << dendl; + + librados::ObjectWriteOperation op; + op.remove(); + + librados::AioCompletion *comp = create_rados_callback< + RemoveImageStateRequest, + &RemoveImageStateRequest::handle_remove_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void RemoveImageStateRequest::handle_remove_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_object_count == 0) { + finish(0); + return; + } + + remove_object(); +} + +template +void RemoveImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::RemoveImageStateRequest; diff --git a/src/librbd/mirror/snapshot/RemoveImageStateRequest.h b/src/librbd/mirror/snapshot/RemoveImageStateRequest.h new file mode 100644 index 000000000..be7dad8e0 --- /dev/null +++ b/src/librbd/mirror/snapshot/RemoveImageStateRequest.h @@ -0,0 +1,75 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H + +#include "include/buffer.h" +#include "include/types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class RemoveImageStateRequest { +public: + static RemoveImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) { + return new RemoveImageStateRequest(image_ctx, snap_id, on_finish); + } + + RemoveImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_OBJECT_COUNT + * | + * v + * REMOVE_OBJECT (repeat for + * | every object) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + Context *m_on_finish; + + bufferlist m_bl; + + size_t m_object_count = 0; + + void get_object_count(); + void handle_get_object_count(int r); + + void remove_object(); + void handle_remove_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::RemoveImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/SetImageStateRequest.cc b/src/librbd/mirror/snapshot/SetImageStateRequest.cc new file mode 100644 index 000000000..9fcee0322 --- /dev/null +++ b/src/librbd/mirror/snapshot/SetImageStateRequest.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/SetImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/image/GetMetadataRequest.h" +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" + +#include + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror_snapshot::SetImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void SetImageStateRequest::send() { + get_name(); +} + +template +void SetImageStateRequest::get_name() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::dir_get_name_start(&op, m_image_ctx->id); + + librados::AioCompletion *comp = create_rados_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_get_name>(this); + m_bl.clear(); + int r = m_image_ctx->md_ctx.aio_operate(RBD_DIRECTORY, comp, &op, &m_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void SetImageStateRequest::handle_get_name(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + auto it = m_bl.cbegin(); + r = cls_client::dir_get_name_finish(&it, &m_image_state.name); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve image name: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ldout(cct, 15) << "name=" << m_image_state.name << dendl; + + get_snap_limit(); +} + +template +void SetImageStateRequest::get_snap_limit() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::snapshot_get_limit_start(&op); + + librados::AioCompletion *comp = create_rados_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_get_snap_limit>(this); + m_bl.clear(); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, comp, &op, + &m_bl); + ceph_assert(r == 0); + comp->release(); +} + +template +void SetImageStateRequest::handle_get_snap_limit(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + auto it = m_bl.cbegin(); + r = cls_client::snapshot_get_limit_finish(&it, &m_image_state.snap_limit); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve snapshot limit: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ldout(cct, 15) << "snap_limit=" << m_image_state.snap_limit << dendl; + + get_metadata(); +} + +template +void SetImageStateRequest::get_metadata() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_get_metadata>(this); + auto req = image::GetMetadataRequest::create( + m_image_ctx->md_ctx, m_image_ctx->header_oid, true, "", "", 0, + &m_image_state.metadata, ctx); + req->send(); +} + +template +void SetImageStateRequest::handle_get_metadata(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to retrieve metadata: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + + m_image_state.features = + m_image_ctx->features & ~RBD_FEATURES_IMPLICIT_ENABLE; + + for (auto &[snap_id, snap_info] : m_image_ctx->snap_info) { + auto type = cls::rbd::get_snap_namespace_type(snap_info.snap_namespace); + if (type != cls::rbd::SNAPSHOT_NAMESPACE_TYPE_USER) { + // only replicate user snapshots -- trash snapshots will be + // replicated by an implicit delete if required + continue; + } + m_image_state.snapshots[snap_id] = {snap_info.snap_namespace, + snap_info.name, + snap_info.protection_status}; + } + } + + write_image_state(); +} + +template +void SetImageStateRequest::write_image_state() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_write_image_state>(this); + + auto req = WriteImageStateRequest::create(m_image_ctx, m_snap_id, + m_image_state, ctx); + req->send(); +} + +template +void SetImageStateRequest::handle_write_image_state(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write image state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + update_primary_snapshot(); +} + +template +void SetImageStateRequest::update_primary_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_snapshot_set_copy_progress( + &op, m_snap_id, true, 0); + + auto aio_comp = create_rados_callback< + SetImageStateRequest, + &SetImageStateRequest::handle_update_primary_snapshot>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void SetImageStateRequest::handle_update_primary_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to update primary snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void SetImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::SetImageStateRequest; diff --git a/src/librbd/mirror/snapshot/SetImageStateRequest.h b/src/librbd/mirror/snapshot/SetImageStateRequest.h new file mode 100644 index 000000000..fd2815494 --- /dev/null +++ b/src/librbd/mirror/snapshot/SetImageStateRequest.h @@ -0,0 +1,96 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H + +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class SetImageStateRequest { +public: + static SetImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) { + return new SetImageStateRequest(image_ctx, snap_id, on_finish); + } + + SetImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * GET_NAME + * | + * v + * GET_SNAP_LIMIT + * | + * v + * GET_METADATA + * | + * v + * WRITE_IMAGE_STATE + * | + * v + * UPDATE_PRIMARY_SNAPSHOT + * | + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + Context *m_on_finish; + + ImageState m_image_state; + + bufferlist m_bl; + bufferlist m_state_bl; + + void get_name(); + void handle_get_name(int r); + + void get_snap_limit(); + void handle_get_snap_limit(int r); + + void get_metadata(); + void handle_get_metadata(int r); + + void write_image_state(); + void handle_write_image_state(int r); + + void update_primary_snapshot(); + void handle_update_primary_snapshot(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::SetImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/Types.cc b/src/librbd/mirror/snapshot/Types.cc new file mode 100644 index 000000000..866b4c3e2 --- /dev/null +++ b/src/librbd/mirror/snapshot/Types.cc @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/Formatter.h" +#include "include/encoding.h" +#include "include/stringify.h" +#include "librbd/mirror/snapshot/Types.h" + +namespace librbd { +namespace mirror { +namespace snapshot { + +void ImageStateHeader::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(object_count, bl); + ENCODE_FINISH(bl); +} + +void ImageStateHeader::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(object_count, bl); + DECODE_FINISH(bl); +} + +void SnapState::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(snap_namespace, bl); + encode(name, bl); + encode(protection_status, bl); + ENCODE_FINISH(bl); +} + +void SnapState::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(snap_namespace, bl); + decode(name, bl); + decode(protection_status, bl); + DECODE_FINISH(bl); +} + +void SnapState::dump(Formatter *f) const { + f->open_object_section("namespace"); + snap_namespace.dump(f); + f->close_section(); + f->dump_string("name", name); + f->dump_unsigned("protection_status", protection_status); +} + +std::ostream& operator<<(std::ostream& os, const SnapState& snap_state) { + os << "[" + << "namespace=" << snap_state.snap_namespace << ", " + << "name=" << snap_state.name << ", " + << "protection=" << static_cast(snap_state.protection_status) + << "]"; + return os; +} + +void ImageState::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(name, bl); + encode(features, bl); + encode(snap_limit, bl); + encode(snapshots, bl); + encode(metadata, bl); + ENCODE_FINISH(bl); +} + +void ImageState::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(name, bl); + decode(features, bl); + decode(snap_limit, bl); + decode(snapshots, bl); + decode(metadata, bl); + DECODE_FINISH(bl); +} + +void ImageState::dump(Formatter *f) const { + f->dump_string("name", name); + f->dump_unsigned("features", features); + f->dump_unsigned("snap_limit", snap_limit); + f->open_array_section("snapshots"); + for (auto &[id, snap_state] : snapshots) { + f->open_object_section(stringify(id).c_str()); + snap_state.dump(f); + f->close_section(); // snap_state + } + f->close_section(); // snapshots + f->open_object_section("metadata"); + for (auto &it : metadata) { + f->dump_stream(it.first.c_str()) << it.second; + } + f->close_section(); // metadata +} + +std::ostream& operator<<(std::ostream& os, const ImageState& image_state) { + os << "[" + << "name=" << image_state.name << ", " + << "features=" << image_state.features << ", " + << "snap_limit=" << image_state.snap_limit << ", " + << "snaps=" << image_state.snapshots << ", " + << "metadata_count=" << image_state.metadata.size() + << "]"; + return os; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd diff --git a/src/librbd/mirror/snapshot/Types.h b/src/librbd/mirror/snapshot/Types.h new file mode 100644 index 000000000..79947a5f8 --- /dev/null +++ b/src/librbd/mirror/snapshot/Types.h @@ -0,0 +1,122 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H + +#include "cls/rbd/cls_rbd_types.h" +#include "include/buffer.h" +#include "include/types.h" + +#include +#include + +namespace librbd { +namespace mirror { +namespace snapshot { + +enum CreatePrimaryFlags { + CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS = (1 << 0), + CREATE_PRIMARY_FLAG_DEMOTED = (1 << 1), + CREATE_PRIMARY_FLAG_FORCE = (1 << 2) +}; + +struct ImageStateHeader { + uint32_t object_count = 0; + + ImageStateHeader() { + } + ImageStateHeader(uint32_t object_count) : object_count(object_count) { + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); +}; + +WRITE_CLASS_ENCODER(ImageStateHeader); + +struct SnapState { + cls::rbd::SnapshotNamespace snap_namespace; + std::string name; + uint8_t protection_status = 0; + + SnapState() { + } + SnapState(const cls::rbd::SnapshotNamespace &snap_namespace, + const std::string &name, uint8_t protection_status) + : snap_namespace(snap_namespace), name(name), + protection_status(protection_status) { + } + + bool operator==(const SnapState& rhs) const { + return snap_namespace == rhs.snap_namespace && + name == rhs.name && protection_status == rhs.protection_status; + } + + bool operator<(const SnapState& rhs) const { + if (snap_namespace != rhs.snap_namespace) { + return snap_namespace < rhs.snap_namespace; + } + if (name != rhs.name) { + return name < rhs.name; + } + return protection_status < rhs.protection_status; + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); + void dump(Formatter *f) const; +}; + +std::ostream& operator<<(std::ostream& os, const SnapState& snap_state); + +WRITE_CLASS_ENCODER(SnapState); + +struct ImageState { + std::string name; + uint64_t features = 0; + uint64_t snap_limit = 0; + std::map snapshots; + std::map metadata; + + ImageState() { + } + ImageState(const std::string &name, uint64_t features, uint64_t snap_limit, + const std::map &snapshots, + const std::map &metadata) + : name(name), features(features), snap_limit(snap_limit), + snapshots(snapshots), metadata(metadata) { + } + + bool operator==(const ImageState& rhs) const { + return name == rhs.name && features == rhs.features && + snap_limit == rhs.snap_limit && snapshots == rhs.snapshots; + } + + bool operator<(const ImageState& rhs) const { + if (name != rhs.name) { + return name < rhs.name; + } + if (features != rhs.features) { + return features < rhs.features; + } + if (snap_limit != rhs.snap_limit) { + return snap_limit < rhs.snap_limit; + } + return snapshots < rhs.snapshots; + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); + void dump(Formatter *f) const; +}; + +std::ostream& operator<<(std::ostream& os, const ImageState& image_state); + +WRITE_CLASS_ENCODER(ImageState); + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc new file mode 100644 index 000000000..35313f627 --- /dev/null +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc @@ -0,0 +1,230 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::UnlinkPeerRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void UnlinkPeerRequest::send() { + if (!m_image_ctx->state->is_refresh_required()) { + unlink_peer(); + return; + } + + refresh_image(); +} + +template +void UnlinkPeerRequest::refresh_image() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template +void UnlinkPeerRequest::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + unlink_peer(); +} + +template +void UnlinkPeerRequest::unlink_peer() { + CephContext *cct = m_image_ctx->cct; + + m_image_ctx->image_lock.lock_shared(); + int r = -ENOENT; + cls::rbd::SnapshotNamespace snap_namespace; + std::string snap_name; + bool have_newer_mirror_snapshot = false; + for (auto snap_it = m_image_ctx->snap_info.find(m_snap_id); + snap_it != m_image_ctx->snap_info.end(); ++snap_it) { + if (snap_it->first == m_snap_id) { + r = 0; + snap_namespace = snap_it->second.snap_namespace; + snap_name = snap_it->second.name; + } else if (std::holds_alternative( + snap_it->second.snap_namespace)) { + ldout(cct, 15) << "located newer mirror snapshot" << dendl; + have_newer_mirror_snapshot = true; + break; + } + } + + if (r == -ENOENT) { + ldout(cct, 15) << "missing snapshot: snap_id=" << m_snap_id << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(r); + return; + } + + auto mirror_ns = std::get_if( + &snap_namespace); + if (mirror_ns == nullptr) { + lderr(cct) << "not mirror snapshot (snap_id=" << m_snap_id << ")" << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(-EINVAL); + return; + } + + // if there is or will be no more peers in the mirror snapshot and we have + // a more recent mirror snapshot, remove the older one + if ((mirror_ns->mirror_peer_uuids.empty() || + (mirror_ns->mirror_peer_uuids.size() == 1 && + mirror_ns->mirror_peer_uuids.count(m_mirror_peer_uuid) != 0)) && + have_newer_mirror_snapshot) { + if (m_allow_remove) { + m_image_ctx->image_lock.unlock_shared(); + remove_snapshot(snap_namespace, snap_name); + return; + } else { + ldout(cct, 15) << "skipping removal of snapshot: snap_id=" << m_snap_id + << ", mirror_peer_uuid=" << m_mirror_peer_uuid + << ", mirror_peer_uuids=" << mirror_ns->mirror_peer_uuids + << dendl; + } + } + + if (mirror_ns->mirror_peer_uuids.count(m_mirror_peer_uuid) == 0) { + ldout(cct, 15) << "no peer to unlink: snap_id=" << m_snap_id + << ", mirror_peer_uuid=" << m_mirror_peer_uuid + << ", mirror_peer_uuids=" << mirror_ns->mirror_peer_uuids + << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(0); + return; + } + + m_image_ctx->image_lock.unlock_shared(); + + ldout(cct, 15) << "snap_id=" << m_snap_id << ", " + << "mirror_peer_uuid=" << m_mirror_peer_uuid << dendl; + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_snapshot_unlink_peer(&op, m_snap_id, + m_mirror_peer_uuid); + auto aio_comp = create_rados_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_unlink_peer>(this); + r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void UnlinkPeerRequest::handle_unlink_peer(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == -ERESTART || r == -ENOENT) { + if (r == -ERESTART) { + ldout(cct, 15) << "unlinking last peer not supported" << dendl; + m_allow_remove = true; + } + refresh_image(); + return; + } + + if (r < 0) { + lderr(cct) << "failed to unlink peer: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + notify_update(); +} + +template +void UnlinkPeerRequest::notify_update() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_notify_update>(this); + m_image_ctx->notify_update(ctx); +} + +template +void UnlinkPeerRequest::handle_notify_update(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == -ENOENT || r == -ETIMEDOUT) { + // non-fatel errors + lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; + } else if (r < 0) { + lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + refresh_image(); +} + +template +void UnlinkPeerRequest::remove_snapshot( + const cls::rbd::SnapshotNamespace& snap_namespace, + const std::string& snap_name) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest, &UnlinkPeerRequest::handle_remove_snapshot>(this); + m_image_ctx->operations->snap_remove(snap_namespace, snap_name, ctx); +} + +template +void UnlinkPeerRequest::handle_remove_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove snapshot: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void UnlinkPeerRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + auto on_finish = m_on_finish; + delete this; + on_finish->complete(r); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::UnlinkPeerRequest; diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.h b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h new file mode 100644 index 000000000..192b40d6e --- /dev/null +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h @@ -0,0 +1,98 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H + +#include "include/buffer.h" +#include "cls/rbd/cls_rbd_client.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class UnlinkPeerRequest { +public: + static UnlinkPeerRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + const std::string &mirror_peer_uuid, + bool allow_remove, Context *on_finish) { + return new UnlinkPeerRequest(image_ctx, snap_id, mirror_peer_uuid, + allow_remove, on_finish); + } + + UnlinkPeerRequest(ImageCtxT *image_ctx, uint64_t snap_id, + const std::string &mirror_peer_uuid, bool allow_remove, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), + m_mirror_peer_uuid(mirror_peer_uuid), m_allow_remove(allow_remove), + m_on_finish(on_finish) { + } + + void send(); + +private: + /* + * @verbatim + * + * + * | + * v + * REFRESH_IMAGE <--------------------------\ + * | ^ (not found | + * | * or last) | + * | * | + * |\---------------> UNLINK_PEER --> NOTIFY_UPDATE + * | (not last peer or + * | no newer mirror + * | snap exists) + * | + * |\---------------> REMOVE_SNAPSHOT + * | (last peer and | + * | newer mirror | + * | snap exists) | + * | | + * |(peer not found) | + * v | + * <---------------/ + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + std::string m_mirror_peer_uuid; + bool m_allow_remove; + Context *m_on_finish; + + void refresh_image(); + void handle_refresh_image(int r); + + void unlink_peer(); + void handle_unlink_peer(int r); + + void notify_update(); + void handle_notify_update(int r); + + void remove_snapshot(const cls::rbd::SnapshotNamespace& snap_namespace, + const std::string& snap_name); + void handle_remove_snapshot(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::UnlinkPeerRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H diff --git a/src/librbd/mirror/snapshot/Utils.cc b/src/librbd/mirror/snapshot/Utils.cc new file mode 100644 index 000000000..36d1558be --- /dev/null +++ b/src/librbd/mirror/snapshot/Utils.cc @@ -0,0 +1,186 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/dout.h" +#include "common/errno.h" +#include "include/stringify.h" +#include "librbd/ImageCtx.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::util: " \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { +namespace util { + +namespace { + +const std::string IMAGE_STATE_OBJECT_PREFIX = "rbd_mirror_snapshot."; + +bool get_rollback_snap_id( + std::map::reverse_iterator it, + std::map::reverse_iterator end, + uint64_t *rollback_snap_id) { + + for (; it != end; it++) { + auto mirror_ns = std::get( + it->second.snap_namespace); + if (mirror_ns.state != cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + break; + } + if (mirror_ns.complete) { + break; + } + } + + if (it != end) { + *rollback_snap_id = it->first; + return true; + } + + return false; +} + +} // anonymous namespace + +std::string get_image_meta_key(const std::string& mirror_uuid) { + return ".rbd_mirror." + mirror_uuid; +} + +template +bool can_create_primary_snapshot(I *image_ctx, bool demoted, bool force, + bool* requires_orphan, + uint64_t *rollback_snap_id) { + CephContext *cct = image_ctx->cct; + + if (requires_orphan != nullptr) { + *requires_orphan = false; + } + if (rollback_snap_id) { + *rollback_snap_id = CEPH_NOSNAP; + } + + std::shared_lock image_locker{image_ctx->image_lock}; + + for (auto it = image_ctx->snap_info.rbegin(); + it != image_ctx->snap_info.rend(); it++) { + auto mirror_ns = std::get_if( + &it->second.snap_namespace); + if (mirror_ns == nullptr) { + continue; + } + ldout(cct, 20) << "previous snapshot snap_id=" << it->first << " " + << *mirror_ns << dendl; + if (mirror_ns->is_demoted() && !force) { + lderr(cct) << "trying to create primary snapshot without force " + << "when previous primary snapshot is demoted" + << dendl; + return false; + } + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + if (!force) { + lderr(cct) << "trying to create primary snapshot without force " + << "when previous snapshot is non-primary" + << dendl; + return false; + } + if (demoted) { + lderr(cct) << "trying to create primary demoted snapshot " + << "when previous snapshot is non-primary" + << dendl; + return false; + } + + if (requires_orphan != nullptr) { + *requires_orphan = !mirror_ns->is_demoted(); + } + if (!mirror_ns->complete) { + ldout(cct, 20) << "needs rollback" << dendl; + if (!rollback_snap_id) { + lderr(cct) << "trying to create primary snapshot " + << "when previous non-primary snapshot is not copied yet" + << dendl; + return false; + } + if (!get_rollback_snap_id(++it, image_ctx->snap_info.rend(), + rollback_snap_id)) { + lderr(cct) << "cannot rollback" << dendl; + return false; + } + ldout(cct, 20) << "rollback_snap_id=" << *rollback_snap_id << dendl; + } + return true; + } + + return true; + } + + ldout(cct, 20) << "no previous mirror snapshots found" << dendl; + return true; +} + +template +bool can_create_non_primary_snapshot(I *image_ctx) { + CephContext *cct = image_ctx->cct; + + std::shared_lock image_locker{image_ctx->image_lock}; + + for (auto it = image_ctx->snap_info.rbegin(); + it != image_ctx->snap_info.rend(); it++) { + auto mirror_ns = std::get_if( + &it->second.snap_namespace); + if (mirror_ns != nullptr) { + ldout(cct, 20) << "previous mirror snapshot snap_id=" << it->first << " " + << *mirror_ns << dendl; + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + if (!mirror_ns->complete) { + lderr(cct) << "trying to create non-primary snapshot " + << "when previous non-primary snapshot is not copied yet" + << dendl; + return false; + } + return true; + } + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(cct) << "trying to create non-primary snapshot " + << "when previous primary snapshot is not in demoted state" + << dendl; + return false; + } + return true; + } + } + + ldout(cct, 20) << "no previous mirror snapshots found" << dendl; + return true; +} + +template +std::string image_state_object_name(I *image_ctx, uint64_t snap_id, + uint64_t index) { + return IMAGE_STATE_OBJECT_PREFIX + image_ctx->id + "." + + stringify(snap_id) + "." + stringify(index); +} + +} // namespace util +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template bool librbd::mirror::snapshot::util::can_create_primary_snapshot( + librbd::ImageCtx *image_ctx, bool demoted, bool force, + bool* requires_orphan, uint64_t *rollback_snap_id); + +template bool librbd::mirror::snapshot::util::can_create_non_primary_snapshot( + librbd::ImageCtx *image_ctx); + +template std::string librbd::mirror::snapshot::util::image_state_object_name( + librbd::ImageCtx *image_ctx, uint64_t snap_id, uint64_t index); diff --git a/src/librbd/mirror/snapshot/Utils.h b/src/librbd/mirror/snapshot/Utils.h new file mode 100644 index 000000000..127ec5865 --- /dev/null +++ b/src/librbd/mirror/snapshot/Utils.h @@ -0,0 +1,38 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H + +#include "include/int_types.h" +#include "include/stringify.h" +#include + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { +namespace util { + +std::string get_image_meta_key(const std::string& mirror_uuid); + +template +bool can_create_primary_snapshot(ImageCtxT *image_ctx, bool demoted, bool force, + bool* requires_orphan, + uint64_t *rollback_snap_id); + +template +bool can_create_non_primary_snapshot(ImageCtxT *image_ctx); + +template +std::string image_state_object_name(ImageCtxT *image_ctx, uint64_t snap_id, + uint64_t index); + +} // namespace util +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H diff --git a/src/librbd/mirror/snapshot/WriteImageStateRequest.cc b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc new file mode 100644 index 000000000..c79dd7e2c --- /dev/null +++ b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc @@ -0,0 +1,120 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::WriteImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +namespace { + +static size_t header_length() { + bufferlist bl; + ImageStateHeader header; + + using ceph::encode; + encode(header, bl); + + return bl.length(); +} + +} +using librbd::util::create_rados_callback; + +template +WriteImageStateRequest::WriteImageStateRequest(I *image_ctx, + uint64_t snap_id, + const ImageState &image_state, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_image_state(image_state), + m_on_finish(on_finish), m_object_size( + 1 << image_ctx->config.template get_val("rbd_default_order")) { + bufferlist bl; + encode(m_image_state, bl); + + m_object_count = 1 + (header_length() + bl.length()) / m_object_size; + ImageStateHeader header(m_object_count); + + encode(header, m_bl); + m_bl.claim_append(bl); +} + +template +void WriteImageStateRequest::send() { + write_object(); +} + +template +void WriteImageStateRequest::write_object() { + CephContext *cct = m_image_ctx->cct; + ceph_assert(m_object_count > 0); + + m_object_count--; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_count); + ldout(cct, 15) << oid << dendl; + + size_t off = m_object_count * m_object_size; + size_t len = std::min(m_bl.length() - off, m_object_size); + bufferlist bl; + bl.substr_of(m_bl, off, len); + + librados::ObjectWriteOperation op; + op.write_full(bl); + + librados::AioCompletion *comp = create_rados_callback< + WriteImageStateRequest, + &WriteImageStateRequest::handle_write_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template +void WriteImageStateRequest::handle_write_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_object_count == 0) { + finish(0); + return; + } + + write_object(); +} + +template +void WriteImageStateRequest::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::WriteImageStateRequest; diff --git a/src/librbd/mirror/snapshot/WriteImageStateRequest.h b/src/librbd/mirror/snapshot/WriteImageStateRequest.h new file mode 100644 index 000000000..d2c4a7f80 --- /dev/null +++ b/src/librbd/mirror/snapshot/WriteImageStateRequest.h @@ -0,0 +1,73 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H + +#include "librbd/mirror/snapshot/Types.h" + +#include +#include + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template +class WriteImageStateRequest { +public: + static WriteImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + const ImageState &image_state, + Context *on_finish) { + return new WriteImageStateRequest(image_ctx, snap_id, image_state, + on_finish); + } + + WriteImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + const ImageState &image_state, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * + * | + * v + * WRITE_OBJECT (repeat for + * | every object) + * v + * + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + ImageState m_image_state; + Context *m_on_finish; + + bufferlist m_bl; + + const size_t m_object_size; + size_t m_object_count = 0; + + void write_object(); + void handle_write_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::WriteImageStateRequest; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H -- cgit v1.2.3