diff options
Diffstat (limited to 'src/librbd/mirror/snapshot')
24 files changed, 3479 insertions, 0 deletions
diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc new file mode 100644 index 000000000..eed0aa506 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.cc @@ -0,0 +1,273 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Utils.h" +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::CreateNonPrimaryRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +CreateNonPrimaryRequest<I>::CreateNonPrimaryRequest( + I* image_ctx, bool demoted, const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, const SnapSeqs& snap_seqs, + const ImageState &image_state, uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_demoted(demoted), + m_primary_mirror_uuid(primary_mirror_uuid), + m_primary_snap_id(primary_snap_id), m_snap_seqs(snap_seqs), + m_image_state(image_state), m_snap_id(snap_id), m_on_finish(on_finish) { + m_default_ns_ctx.dup(m_image_ctx->md_ctx); + m_default_ns_ctx.set_namespace(""); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::send() { + refresh_image(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::refresh_image() { + if (!m_image_ctx->state->is_refresh_required()) { + get_mirror_image(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest<I>, + &CreateNonPrimaryRequest<I>::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_mirror_image(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::get_mirror_image() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_image_get_start(&op, m_image_ctx->id); + + librados::AioCompletion *comp = create_rados_callback< + CreateNonPrimaryRequest<I>, + &CreateNonPrimaryRequest<I>::handle_get_mirror_image>(this); + int r = m_image_ctx->md_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::handle_get_mirror_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + cls::rbd::MirrorImage mirror_image; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_image_get_finish(&iter, &mirror_image); + } + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to retrieve mirroring state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (mirror_image.mode != cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + lderr(cct) << "snapshot based mirroring is not enabled" << dendl; + finish(-EINVAL); + return; + } + + if (!is_orphan() && !util::can_create_non_primary_snapshot(m_image_ctx)) { + finish(-EINVAL); + return; + } + + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_snap_name = ".mirror.non_primary." + mirror_image.global_image_id + "." + + uuid_gen.to_string(); + + get_mirror_peers(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::get_mirror_peers() { + if (!m_demoted) { + create_snapshot(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + auto aio_comp = create_rados_callback< + CreateNonPrimaryRequest<I>, + &CreateNonPrimaryRequest<I>::handle_get_mirror_peers>(this); + m_out_bl.clear(); + int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::handle_get_mirror_peers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + std::vector<cls::rbd::MirrorPeer> peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + + create_snapshot(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::create_snapshot() { + CephContext *cct = m_image_ctx->cct; + + cls::rbd::MirrorSnapshotNamespace ns{ + (m_demoted ? cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED : + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY), {}, + m_primary_mirror_uuid, m_primary_snap_id}; + if (m_demoted) { + ns.mirror_peer_uuids = m_mirror_peer_uuids; + } + ns.snap_seqs = m_snap_seqs; + ns.complete = is_orphan(); + ldout(cct, 15) << "ns=" << ns << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest<I>, + &CreateNonPrimaryRequest<I>::handle_create_snapshot>(this); + m_image_ctx->operations->snap_create(ns, m_snap_name, 0, m_prog_ctx, ctx); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + write_image_state(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::write_image_state() { + uint64_t snap_id; + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + snap_id = m_image_ctx->get_snap_id( + cls::rbd::MirrorSnapshotNamespace{}, m_snap_name); + } + + if (m_snap_id != nullptr) { + *m_snap_id = snap_id; + } + + if (is_orphan()) { + finish(0); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreateNonPrimaryRequest<I>, + &CreateNonPrimaryRequest<I>::handle_write_image_state>(this); + + auto req = WriteImageStateRequest<I>::create(m_image_ctx, snap_id, + m_image_state, ctx); + req->send(); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::handle_write_image_state(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write image state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void CreateNonPrimaryRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::CreateNonPrimaryRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h new file mode 100644 index 000000000..36f155413 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreateNonPrimaryRequest.h @@ -0,0 +1,123 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H + +#include "include/buffer.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/Types.h" +#include "librbd/internal.h" +#include "librbd/mirror/snapshot/Types.h" + +#include <string> +#include <set> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class CreateNonPrimaryRequest { +public: + static CreateNonPrimaryRequest *create(ImageCtxT *image_ctx, + bool demoted, + const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, + const SnapSeqs& snap_seqs, + const ImageState &image_state, + uint64_t *snap_id, + Context *on_finish) { + return new CreateNonPrimaryRequest(image_ctx, demoted, primary_mirror_uuid, + primary_snap_id, snap_seqs, image_state, + snap_id, on_finish); + } + + CreateNonPrimaryRequest(ImageCtxT *image_ctx, + bool demoted, + const std::string &primary_mirror_uuid, + uint64_t primary_snap_id, + const SnapSeqs& snap_seqs, + const ImageState &image_state, uint64_t *snap_id, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * REFRESH_IMAGE + * | + * v + * GET_MIRROR_IMAGE + * | + * v (skip if not needed) + * GET_MIRROR_PEERS + * | + * v + * CREATE_SNAPSHOT + * | + * v + * WRITE_IMAGE_STATE + * | + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + bool m_demoted; + std::string m_primary_mirror_uuid; + uint64_t m_primary_snap_id; + SnapSeqs m_snap_seqs; + ImageState m_image_state; + uint64_t *m_snap_id; + Context *m_on_finish; + + librados::IoCtx m_default_ns_ctx; + std::set<std::string> m_mirror_peer_uuids; + + std::string m_snap_name; + + bufferlist m_out_bl; + NoOpProgressContext m_prog_ctx; + + bool is_orphan() const { + return m_primary_mirror_uuid.empty(); + } + + void refresh_image(); + void handle_refresh_image(int r); + + void get_mirror_image(); + void handle_get_mirror_image(int r); + + void get_mirror_peers(); + void handle_get_mirror_peers(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void write_image_state(); + void handle_write_image_state(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::CreateNonPrimaryRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_NON_PRIMARY_REQUEST_H diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc new file mode 100644 index 000000000..54da9ad61 --- /dev/null +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.cc @@ -0,0 +1,277 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::CreatePrimaryRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +CreatePrimaryRequest<I>::CreatePrimaryRequest( + I *image_ctx, const std::string& global_image_id, + uint64_t clean_since_snap_id, uint64_t snap_create_flags, uint32_t flags, + uint64_t *snap_id, Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_clean_since_snap_id(clean_since_snap_id), + m_snap_create_flags(snap_create_flags), m_flags(flags), m_snap_id(snap_id), + m_on_finish(on_finish) { + m_default_ns_ctx.dup(m_image_ctx->md_ctx); + m_default_ns_ctx.set_namespace(""); +} + +template <typename I> +void CreatePrimaryRequest<I>::send() { + if (!util::can_create_primary_snapshot( + m_image_ctx, + ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0), + ((m_flags & CREATE_PRIMARY_FLAG_FORCE) != 0), nullptr, nullptr)) { + finish(-EINVAL); + return; + } + + uuid_d uuid_gen; + uuid_gen.generate_random(); + m_snap_name = ".mirror.primary." + m_global_image_id + "." + + uuid_gen.to_string(); + + get_mirror_peers(); +} + +template <typename I> +void CreatePrimaryRequest<I>::get_mirror_peers() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::mirror_peer_list_start(&op); + + librados::AioCompletion *comp = create_rados_callback< + CreatePrimaryRequest<I>, + &CreatePrimaryRequest<I>::handle_get_mirror_peers>(this); + m_out_bl.clear(); + int r = m_default_ns_ctx.aio_operate(RBD_MIRRORING, comp, &op, &m_out_bl); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void CreatePrimaryRequest<I>::handle_get_mirror_peers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + std::vector<cls::rbd::MirrorPeer> peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve mirror peers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + for (auto &peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + m_mirror_peer_uuids.insert(peer.uuid); + } + + if (m_mirror_peer_uuids.empty() && + ((m_flags & CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS) == 0)) { + lderr(cct) << "no mirror tx peers configured for the pool" << dendl; + finish(-EINVAL); + return; + } + + create_snapshot(); +} + +template <typename I> +void CreatePrimaryRequest<I>::create_snapshot() { + cls::rbd::MirrorSnapshotNamespace ns{ + ((m_flags & CREATE_PRIMARY_FLAG_DEMOTED) != 0 ? + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED : + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY), + m_mirror_peer_uuids, "", m_clean_since_snap_id}; + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "name=" << m_snap_name << ", " + << "ns=" << ns << dendl; + auto ctx = create_context_callback< + CreatePrimaryRequest<I>, + &CreatePrimaryRequest<I>::handle_create_snapshot>(this); + m_image_ctx->operations->snap_create(ns, m_snap_name, m_snap_create_flags, + m_prog_ctx, ctx); +} + +template <typename I> +void CreatePrimaryRequest<I>::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + refresh_image(); +} + +template <typename I> +void CreatePrimaryRequest<I>::refresh_image() { + // if snapshot created via remote RPC, refresh is required to retrieve + // the snapshot id + if (m_snap_id == nullptr) { + unlink_peer(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + CreatePrimaryRequest<I>, + &CreatePrimaryRequest<I>::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template <typename I> +void CreatePrimaryRequest<I>::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + *m_snap_id = m_image_ctx->get_snap_id( + cls::rbd::MirrorSnapshotNamespace{}, m_snap_name); + ldout(cct, 15) << "snap_id=" << *m_snap_id << dendl; + } + + unlink_peer(); +} + +template <typename I> +void CreatePrimaryRequest<I>::unlink_peer() { + uint64_t max_snapshots = m_image_ctx->config.template get_val<uint64_t>( + "rbd_mirroring_max_mirroring_snapshots"); + ceph_assert(max_snapshots >= 3); + + std::string peer_uuid; + uint64_t snap_id = CEPH_NOSNAP; + + for (auto &peer : m_mirror_peer_uuids) { + std::shared_lock image_locker{m_image_ctx->image_lock}; + size_t count = 0; + uint64_t unlink_snap_id = 0; + for (auto &snap_it : m_image_ctx->snap_info) { + auto info = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &snap_it.second.snap_namespace); + if (info == nullptr) { + continue; + } + if (info->state != cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + // reset counters -- we count primary snapshots after the last promotion + count = 0; + unlink_snap_id = 0; + continue; + } + // call UnlinkPeerRequest only if the snapshot is linked with this peer + // or if it's not linked with any peer (happens if mirroring is enabled + // on a pool with no peers configured or if UnlinkPeerRequest gets + // interrupted) + if (info->mirror_peer_uuids.size() == 0) { + peer_uuid = peer; + snap_id = snap_it.first; + break; + } + if (info->mirror_peer_uuids.count(peer) == 0) { + continue; + } + count++; + if (count == max_snapshots) { + unlink_snap_id = snap_it.first; + } + if (count > max_snapshots) { + peer_uuid = peer; + snap_id = unlink_snap_id; + break; + } + } + if (snap_id != CEPH_NOSNAP) { + break; + } + } + + if (snap_id == CEPH_NOSNAP) { + finish(0); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "peer=" << peer_uuid << ", snap_id=" << snap_id << dendl; + + auto ctx = create_context_callback< + CreatePrimaryRequest<I>, + &CreatePrimaryRequest<I>::handle_unlink_peer>(this); + auto req = UnlinkPeerRequest<I>::create(m_image_ctx, snap_id, peer_uuid, ctx); + req->send(); +} + +template <typename I> +void CreatePrimaryRequest<I>::handle_unlink_peer(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to unlink peer: " << cpp_strerror(r) << dendl; + finish(0); // not fatal + return; + } + + unlink_peer(); +} + +template <typename I> +void CreatePrimaryRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::CreatePrimaryRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/CreatePrimaryRequest.h b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h new file mode 100644 index 000000000..b8e84cf2b --- /dev/null +++ b/src/librbd/mirror/snapshot/CreatePrimaryRequest.h @@ -0,0 +1,106 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/internal.h" +#include "librbd/mirror/snapshot/Types.h" + +#include <string> +#include <set> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class CreatePrimaryRequest { +public: + static CreatePrimaryRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + uint64_t clean_since_snap_id, + uint64_t snap_create_flags, + uint32_t flags, uint64_t *snap_id, + Context *on_finish) { + return new CreatePrimaryRequest(image_ctx, global_image_id, + clean_since_snap_id, snap_create_flags, flags, + snap_id, on_finish); + } + + CreatePrimaryRequest(ImageCtxT *image_ctx, + const std::string& global_image_id, + uint64_t clean_since_snap_id, uint64_t snap_create_flags, + uint32_t flags, uint64_t *snap_id, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * GET_MIRROR_PEERS + * | + * v + * CREATE_SNAPSHOT + * | + * v + * REFRESH_IMAGE + * | + * v + * UNLINK_PEER (skip if not needed, + * | repeat if needed) + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + uint64_t m_clean_since_snap_id; + const uint64_t m_snap_create_flags; + const uint32_t m_flags; + uint64_t *m_snap_id; + Context *m_on_finish; + + librados::IoCtx m_default_ns_ctx; + std::set<std::string> m_mirror_peer_uuids; + std::string m_snap_name; + + bufferlist m_out_bl; + NoOpProgressContext m_prog_ctx; + + void get_mirror_peers(); + void handle_get_mirror_peers(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void refresh_image(); + void handle_refresh_image(int r); + + void unlink_peer(); + void handle_unlink_peer(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::CreatePrimaryRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_CREATE_PRIMARY_REQUEST_H diff --git a/src/librbd/mirror/snapshot/DemoteRequest.cc b/src/librbd/mirror/snapshot/DemoteRequest.cc new file mode 100644 index 000000000..ccaa33c83 --- /dev/null +++ b/src/librbd/mirror/snapshot/DemoteRequest.cc @@ -0,0 +1,110 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/DemoteRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::DemoteRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +void DemoteRequest<I>::send() { + enable_non_primary_feature(); +} + +template <typename I> +void DemoteRequest<I>::enable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // ensure image is flagged with non-primary feature so that + // standard RBD clients cannot write to it. + librados::ObjectWriteOperation op; + cls_client::set_features(&op, RBD_FEATURE_NON_PRIMARY, + RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + DemoteRequest<I>, + &DemoteRequest<I>::handle_enable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void DemoteRequest<I>::handle_enable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to enable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_snapshot(); +} + +template <typename I> +void DemoteRequest<I>::create_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + DemoteRequest<I>, &DemoteRequest<I>::handle_create_snapshot>(this); + + auto req = CreatePrimaryRequest<I>::create( + m_image_ctx, m_global_image_id, CEPH_NOSNAP, + SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, + (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | + snapshot::CREATE_PRIMARY_FLAG_DEMOTED), nullptr, ctx); + req->send(); +} + +template <typename I> +void DemoteRequest<I>::handle_create_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create mirror snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void DemoteRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::DemoteRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/DemoteRequest.h b/src/librbd/mirror/snapshot/DemoteRequest.h new file mode 100644 index 000000000..63c935645 --- /dev/null +++ b/src/librbd/mirror/snapshot/DemoteRequest.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H + +#include "include/buffer.h" + +#include <string> +#include <set> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class DemoteRequest { +public: + static DemoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + Context *on_finish) { + return new DemoteRequest(image_ctx, global_image_id, on_finish); + } + + DemoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * ENABLE_NON_PRIMARY_FEATURE + * | + * v + * CREATE_SNAPSHOT + * | + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + Context *m_on_finish; + + void enable_non_primary_feature(); + void handle_enable_non_primary_feature(int r); + + void create_snapshot(); + void handle_create_snapshot(int r); + + void finish(int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::DemoteRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_DEMOTE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/GetImageStateRequest.cc b/src/librbd/mirror/snapshot/GetImageStateRequest.cc new file mode 100644 index 000000000..4692f88cb --- /dev/null +++ b/src/librbd/mirror/snapshot/GetImageStateRequest.cc @@ -0,0 +1,114 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/GetImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Types.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::GetImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + +template <typename I> +void GetImageStateRequest<I>::send() { + read_object(); +} + + +template <typename I> +void GetImageStateRequest<I>::read_object() { + CephContext *cct = m_image_ctx->cct; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_index); + ldout(cct, 15) << oid << dendl; + + librados::ObjectReadOperation op; + m_bl.clear(); + op.read(0, 0, &m_bl, nullptr); + + librados::AioCompletion *comp = create_rados_callback< + GetImageStateRequest<I>, + &GetImageStateRequest<I>::handle_read_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op, nullptr); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void GetImageStateRequest<I>::handle_read_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to read image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + auto iter = m_bl.cbegin(); + + if (m_object_index == 0) { + ImageStateHeader header; + try { + using ceph::decode; + decode(header, iter); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state object header" << dendl; + finish(-EBADMSG); + return; + } + m_object_count = header.object_count; + } + + bufferlist bl; + bl.substr_of(m_bl, iter.get_off(), m_bl.length() - iter.get_off()); + m_state_bl.claim_append(bl); + + m_object_index++; + + if (m_object_index >= m_object_count) { + finish(0); + return; + } + + read_object(); +} + +template <typename I> +void GetImageStateRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + try { + using ceph::decode; + decode(*m_image_state, m_state_bl); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state" << dendl; + r = -EBADMSG; + } + } + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::GetImageStateRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/GetImageStateRequest.h b/src/librbd/mirror/snapshot/GetImageStateRequest.h new file mode 100644 index 000000000..483e3a228 --- /dev/null +++ b/src/librbd/mirror/snapshot/GetImageStateRequest.h @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H + +#include "include/buffer.h" +#include "include/types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +struct ImageState; + +template <typename ImageCtxT = librbd::ImageCtx> +class GetImageStateRequest { +public: + static GetImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + ImageState *image_state, + Context *on_finish) { + return new GetImageStateRequest(image_ctx, snap_id, image_state, on_finish); + } + + GetImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + ImageState *image_state, Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_image_state(image_state), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * READ_OBJECT (repeat for + * | every object) + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + ImageState *m_image_state; + Context *m_on_finish; + + bufferlist m_bl; + bufferlist m_state_bl; + + size_t m_object_count = 0; + size_t m_object_index = 0; + + void read_object(); + void handle_read_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::GetImageStateRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_GET_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/ImageMeta.cc b/src/librbd/mirror/snapshot/ImageMeta.cc new file mode 100644 index 000000000..826899775 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.cc @@ -0,0 +1,175 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "json_spirit/json_spirit.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/WatchNotifyTypes.h" +#include "librbd/mirror/snapshot/Utils.h" +#include "librbd/watcher/Notifier.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::ImageMeta: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; +using librbd::mirror::snapshot::util::get_image_meta_key; + +template <typename I> +ImageMeta<I>::ImageMeta(I* image_ctx, const std::string& mirror_uuid) + : m_image_ctx(image_ctx), m_mirror_uuid(mirror_uuid) { +} + +template <typename I> +void ImageMeta<I>::load(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + librados::ObjectReadOperation op; + cls_client::metadata_get_start(&op, get_image_meta_key(m_mirror_uuid)); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_load(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void ImageMeta<I>::handle_load(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + std::string data; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = cls_client::metadata_get_finish(&it, &data); + } + + if (r == -ENOENT) { + ldout(m_image_ctx->cct, 15) << "no snapshot-based mirroring image-meta: " + << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } else if (r < 0) { + lderr(m_image_ctx->cct) << "failed to load snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + bool json_valid = false; + json_spirit::mValue json_root; + if (json_spirit::read(data, json_root)) { + try { + auto& json_obj = json_root.get_obj(); + resync_requested = json_obj["resync_requested"].get_bool(); + json_valid = true; + } catch (std::runtime_error&) { + } + } + + if (!json_valid) { + lderr(m_image_ctx->cct) << "invalid image-meta JSON received" << dendl; + on_finish->complete(-EBADMSG); + return; + } + + on_finish->complete(0); +} + +template <typename I> +void ImageMeta<I>::save(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << "oid=" << m_image_ctx->header_oid << ", " + << "key=" << get_image_meta_key(m_mirror_uuid) + << dendl; + + // simple implementation for now + std::string json = "{\"resync_requested\": " + + std::string(resync_requested ? "true" : "false") + "}"; + + bufferlist bl; + bl.append(json); + + // avoid using built-in metadata_set operation since that would require + // opening the non-primary image in read/write mode which isn't supported + librados::ObjectWriteOperation op; + cls_client::metadata_set(&op, {{get_image_meta_key(m_mirror_uuid), bl}}); + + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_save(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void ImageMeta<I>::handle_save(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to save snapshot-based mirroring " + << "image-meta: " << cpp_strerror(r) << dendl; + on_finish->complete(r); + return; + } + + notify_update(on_finish); +} + +template <typename I> +void ImageMeta<I>::notify_update(Context* on_finish) { + ldout(m_image_ctx->cct, 15) << dendl; + + // directly send header notification on image since you cannot + // open a non-primary image read/write and therefore cannot re-use + // the ImageWatcher to send the notification + bufferlist bl; + encode(watch_notify::NotifyMessage(new watch_notify::HeaderUpdatePayload()), + bl); + + m_out_bl.clear(); + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_notify_update(on_finish, r); + }); + auto aio_comp = create_rados_callback(ctx); + int r = m_image_ctx->md_ctx.aio_notify( + m_image_ctx->header_oid, aio_comp, bl, watcher::Notifier::NOTIFY_TIMEOUT, + &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void ImageMeta<I>::handle_notify_update(Context* on_finish, int r) { + ldout(m_image_ctx->cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(m_image_ctx->cct) << "failed to notify image update: " + << cpp_strerror(r) << dendl; + } + on_finish->complete(r); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::ImageMeta<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/ImageMeta.h b/src/librbd/mirror/snapshot/ImageMeta.h new file mode 100644 index 000000000..5d05f1927 --- /dev/null +++ b/src/librbd/mirror/snapshot/ImageMeta.h @@ -0,0 +1,78 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H + +#include "include/rados/librados.hpp" +#include <string> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT> +class ImageMeta { +public: + static ImageMeta* create(ImageCtxT* image_ctx, + const std::string& mirror_uuid) { + return new ImageMeta(image_ctx, mirror_uuid); + } + + ImageMeta(ImageCtxT* image_ctx, const std::string& mirror_uuid); + + void load(Context* on_finish); + void save(Context* on_finish); + + bool resync_requested = false; + +private: + /** + * @verbatim + * + * <start> + * | + * v + * METADATA_GET + * | + * v + * <idle> + * | + * v + * METADATA_SET + * | + * v + * NOTIFY_UPDATE + * | + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT* m_image_ctx; + std::string m_mirror_uuid; + + bufferlist m_out_bl; + + void handle_load(Context* on_finish, int r); + + void handle_save(Context* on_finish, int r); + + void notify_update(Context* on_finish); + void handle_notify_update(Context* on_finish, int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::ImageMeta<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_IMAGE_META_H diff --git a/src/librbd/mirror/snapshot/PromoteRequest.cc b/src/librbd/mirror/snapshot/PromoteRequest.cc new file mode 100644 index 000000000..9718c299e --- /dev/null +++ b/src/librbd/mirror/snapshot/PromoteRequest.cc @@ -0,0 +1,405 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/PromoteRequest.h" +#include "common/Timer.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/image/ListWatchersRequest.h" +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "librbd/mirror/snapshot/CreatePrimaryRequest.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::PromoteRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +void PromoteRequest<I>::send() { + CephContext *cct = m_image_ctx->cct; + bool requires_orphan = false; + if (!util::can_create_primary_snapshot(m_image_ctx, false, true, + &requires_orphan, + &m_rollback_snap_id)) { + lderr(cct) << "cannot promote" << dendl; + finish(-EINVAL); + return; + } else if (m_rollback_snap_id == CEPH_NOSNAP && !requires_orphan) { + create_promote_snapshot(); + return; + } + + ldout(cct, 15) << "requires_orphan=" << requires_orphan << ", " + << "rollback_snap_id=" << m_rollback_snap_id << dendl; + create_orphan_snapshot(); +} + +template <typename I> +void PromoteRequest<I>::create_orphan_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest<I>, + &PromoteRequest<I>::handle_create_orphan_snapshot>(this); + + auto req = CreateNonPrimaryRequest<I>::create( + m_image_ctx, false, "", CEPH_NOSNAP, {}, {}, nullptr, ctx); + req->send(); +} + +template <typename I> +void PromoteRequest<I>::handle_create_orphan_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create orphan snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + list_watchers(); +} + +template <typename I> +void PromoteRequest<I>::list_watchers() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest<I>, + &PromoteRequest<I>::handle_list_watchers>(this); + + m_watchers.clear(); + auto flags = librbd::image::LIST_WATCHERS_FILTER_OUT_MY_INSTANCE | + librbd::image::LIST_WATCHERS_MIRROR_INSTANCES_ONLY; + auto req = librbd::image::ListWatchersRequest<I>::create( + *m_image_ctx, flags, &m_watchers, ctx); + req->send(); +} + +template <typename I> +void PromoteRequest<I>::handle_list_watchers(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to list watchers: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_watchers.empty()) { + acquire_exclusive_lock(); + return; + } + + wait_update_notify(); +} + +template <typename I> +void PromoteRequest<I>::wait_update_notify() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + ImageCtx::get_timer_instance(cct, &m_timer, &m_timer_lock); + + std::lock_guard timer_lock{*m_timer_lock}; + + m_scheduler_ticks = 5; + + int r = m_image_ctx->state->register_update_watcher(&m_update_watch_ctx, + &m_update_watcher_handle); + if (r < 0) { + lderr(cct) << "failed to register update watcher: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + scheduler_unregister_update_watcher(); +} + +template <typename I> +void PromoteRequest<I>::handle_update_notify() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + std::lock_guard timer_lock{*m_timer_lock}; + m_scheduler_ticks = 0; +} + +template <typename I> +void PromoteRequest<I>::scheduler_unregister_update_watcher() { + ceph_assert(ceph_mutex_is_locked(*m_timer_lock)); + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "scheduler_ticks=" << m_scheduler_ticks << dendl; + + if (m_scheduler_ticks > 0) { + m_scheduler_ticks--; + m_timer->add_event_after(1, new LambdaContext([this](int) { + scheduler_unregister_update_watcher(); + })); + return; + } + + m_image_ctx->op_work_queue->queue(new LambdaContext([this](int) { + unregister_update_watcher(); + }), 0); +} + +template <typename I> +void PromoteRequest<I>::unregister_update_watcher() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest<I>, + &PromoteRequest<I>::handle_unregister_update_watcher>(this); + + m_image_ctx->state->unregister_update_watcher(m_update_watcher_handle, ctx); +} + +template <typename I> +void PromoteRequest<I>::handle_unregister_update_watcher(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to unregister update watcher: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + list_watchers(); +} + +template <typename I> +void PromoteRequest<I>::acquire_exclusive_lock() { + { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr && + !m_image_ctx->exclusive_lock->is_lock_owner()) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + m_lock_acquired = true; + m_image_ctx->exclusive_lock->block_requests(0); + + auto ctx = create_context_callback< + PromoteRequest<I>, + &PromoteRequest<I>::handle_acquire_exclusive_lock>(this); + + m_image_ctx->exclusive_lock->acquire_lock(ctx); + return; + } + } + + rollback(); +} + +template <typename I> +void PromoteRequest<I>::handle_acquire_exclusive_lock(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to acquire exclusive lock: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } else { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr && + !m_image_ctx->exclusive_lock->is_lock_owner()) { + lderr(cct) << "failed to acquire exclusive lock" << dendl; + r = m_image_ctx->exclusive_lock->get_unlocked_op_error(); + locker.unlock(); + finish(r); + return; + } + } + + rollback(); +} + +template <typename I> +void PromoteRequest<I>::rollback() { + if (m_rollback_snap_id == CEPH_NOSNAP) { + create_promote_snapshot(); + return; + } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + std::shared_lock owner_locker{m_image_ctx->owner_lock}; + std::shared_lock image_locker{m_image_ctx->image_lock}; + + auto info = m_image_ctx->get_snap_info(m_rollback_snap_id); + ceph_assert(info != nullptr); + auto snap_namespace = info->snap_namespace; + auto snap_name = info->name; + + image_locker.unlock(); + + auto ctx = create_async_context_callback( + *m_image_ctx, create_context_callback< + PromoteRequest<I>, &PromoteRequest<I>::handle_rollback>(this)); + + m_image_ctx->operations->execute_snap_rollback(snap_namespace, snap_name, + m_progress_ctx, ctx); +} + +template <typename I> +void PromoteRequest<I>::handle_rollback(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to rollback: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + create_promote_snapshot(); +} + +template <typename I> +void PromoteRequest<I>::create_promote_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + PromoteRequest<I>, + &PromoteRequest<I>::handle_create_promote_snapshot>(this); + + auto req = CreatePrimaryRequest<I>::create( + m_image_ctx, m_global_image_id, CEPH_NOSNAP, + SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, + (snapshot::CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS | + snapshot::CREATE_PRIMARY_FLAG_FORCE), nullptr, ctx); + req->send(); +} + +template <typename I> +void PromoteRequest<I>::handle_create_promote_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to create promote snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + disable_non_primary_feature(); +} + +template <typename I> +void PromoteRequest<I>::disable_non_primary_feature() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << dendl; + + // remove the non-primary feature flag so that the image can be + // R/W by standard RBD clients + librados::ObjectWriteOperation op; + cls_client::set_features(&op, 0U, RBD_FEATURE_NON_PRIMARY); + + auto aio_comp = create_rados_callback< + PromoteRequest<I>, + &PromoteRequest<I>::handle_disable_non_primary_feature>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void PromoteRequest<I>::handle_disable_non_primary_feature(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to disable non-primary feature: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + release_exclusive_lock(); +} + +template <typename I> +void PromoteRequest<I>::release_exclusive_lock() { + if (m_lock_acquired) { + std::unique_lock locker{m_image_ctx->owner_lock}; + if (m_image_ctx->exclusive_lock != nullptr) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + m_image_ctx->exclusive_lock->unblock_requests(); + + auto ctx = create_context_callback< + PromoteRequest<I>, + &PromoteRequest<I>::handle_release_exclusive_lock>(this); + + m_image_ctx->exclusive_lock->release_lock(ctx); + return; + } + } + + finish(0); +} + +template <typename I> +void PromoteRequest<I>::handle_release_exclusive_lock(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to release exclusive lock: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void PromoteRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::PromoteRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/PromoteRequest.h b/src/librbd/mirror/snapshot/PromoteRequest.h new file mode 100644 index 000000000..1d9a862a0 --- /dev/null +++ b/src/librbd/mirror/snapshot/PromoteRequest.h @@ -0,0 +1,151 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H + +#include "include/buffer.h" +#include "include/rbd/librbd.hpp" +#include "common/ceph_mutex.h" +#include "common/Timer.h" +#include "librbd/internal.h" + +#include <string> +#include <set> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class PromoteRequest { +public: + static PromoteRequest *create(ImageCtxT *image_ctx, + const std::string& global_image_id, + Context *on_finish) { + return new PromoteRequest(image_ctx, global_image_id, on_finish); + } + + PromoteRequest(ImageCtxT *image_ctx, const std::string& global_image_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_global_image_id(global_image_id), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * | (can promote) + * |\----------------------------------------\ + * | | + * | | + * v (skip if not needed) | + * CREATE_ORPHAN_SNAPSHOT | + * | | + * | /-- UNREGISTER_UPDATE_WATCHER <-\ | + * v v | | + * LIST_WATCHERS ----> WAIT_UPDATE_NOTIFY --/ | + * | | + * | (no watchers) | + * v | + * ACQUIRE_EXCLUSIVE_LOCK | + * | (skip if not needed) | + * v | + * ROLLBACK | + * | | + * v | + * CREATE_PROMOTE_SNAPSHOT <--------------------/ + * | + * v + * DISABLE_NON_PRIMARY_FEATURE + * | + * v + * RELEASE_EXCLUSIVE_LOCK (skip if not needed) + * | + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + std::string m_global_image_id; + Context *m_on_finish; + + uint64_t m_rollback_snap_id = CEPH_NOSNAP; + bool m_lock_acquired = false; + NoOpProgressContext m_progress_ctx; + + class UpdateWatchCtx : public librbd::UpdateWatchCtx { + public: + UpdateWatchCtx(PromoteRequest *promote_request) + : promote_request(promote_request) { + } + + void handle_notify() { + promote_request->handle_update_notify(); + } + + private: + PromoteRequest *promote_request; + + } m_update_watch_ctx = {this}; + + std::list<obj_watch_t> m_watchers; + uint64_t m_update_watcher_handle = 0; + uint64_t m_scheduler_ticks = 0; + SafeTimer *m_timer = nullptr; + ceph::mutex *m_timer_lock = nullptr; + + void refresh_image(); + void handle_refresh_image(int r); + + void create_orphan_snapshot(); + void handle_create_orphan_snapshot(int r); + + void list_watchers(); + void handle_list_watchers(int r); + + void wait_update_notify(); + void handle_update_notify(); + void scheduler_unregister_update_watcher(); + + void unregister_update_watcher(); + void handle_unregister_update_watcher(int r); + + void acquire_exclusive_lock(); + void handle_acquire_exclusive_lock(int r); + + void rollback(); + void handle_rollback(int r); + + void create_promote_snapshot(); + void handle_create_promote_snapshot(int r); + + void disable_non_primary_feature(); + void handle_disable_non_primary_feature(int r); + + void release_exclusive_lock(); + void handle_release_exclusive_lock(int r); + + void finish(int r); + +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::PromoteRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_PROMOTE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc new file mode 100644 index 000000000..204e0489a --- /dev/null +++ b/src/librbd/mirror/snapshot/RemoveImageStateRequest.cc @@ -0,0 +1,131 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/RemoveImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Types.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::RemoveImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_rados_callback; + +template <typename I> +void RemoveImageStateRequest<I>::send() { + get_object_count(); +} + + +template <typename I> +void RemoveImageStateRequest<I>::get_object_count() { + CephContext *cct = m_image_ctx->cct; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, 0); + ldout(cct, 15) << oid << dendl; + + librados::ObjectReadOperation op; + op.read(0, 0, &m_bl, nullptr); + + librados::AioCompletion *comp = create_rados_callback< + RemoveImageStateRequest<I>, + &RemoveImageStateRequest<I>::handle_get_object_count>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op, nullptr); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void RemoveImageStateRequest<I>::handle_get_object_count(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to read image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ImageStateHeader header(1); + auto iter = m_bl.cbegin(); + try { + using ceph::decode; + + decode(header, iter); + } catch (const buffer::error &err) { + lderr(cct) << "failed to decode image state object header" << dendl; + // still try to remove it + } + + m_object_count = header.object_count > 0 ? header.object_count : 1; + + remove_object(); +} + +template <typename I> +void RemoveImageStateRequest<I>::remove_object() { + CephContext *cct = m_image_ctx->cct; + + ceph_assert(m_object_count > 0); + m_object_count--; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_count); + ldout(cct, 15) << oid << dendl; + + librados::ObjectWriteOperation op; + op.remove(); + + librados::AioCompletion *comp = create_rados_callback< + RemoveImageStateRequest<I>, + &RemoveImageStateRequest<I>::handle_remove_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void RemoveImageStateRequest<I>::handle_remove_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove image state object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_object_count == 0) { + finish(0); + return; + } + + remove_object(); +} + +template <typename I> +void RemoveImageStateRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::RemoveImageStateRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/RemoveImageStateRequest.h b/src/librbd/mirror/snapshot/RemoveImageStateRequest.h new file mode 100644 index 000000000..be7dad8e0 --- /dev/null +++ b/src/librbd/mirror/snapshot/RemoveImageStateRequest.h @@ -0,0 +1,75 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H + +#include "include/buffer.h" +#include "include/types.h" + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class RemoveImageStateRequest { +public: + static RemoveImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) { + return new RemoveImageStateRequest(image_ctx, snap_id, on_finish); + } + + RemoveImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * GET_OBJECT_COUNT + * | + * v + * REMOVE_OBJECT (repeat for + * | every object) + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + Context *m_on_finish; + + bufferlist m_bl; + + size_t m_object_count = 0; + + void get_object_count(); + void handle_get_object_count(int r); + + void remove_object(); + void handle_remove_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::RemoveImageStateRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_REMOVE_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/SetImageStateRequest.cc b/src/librbd/mirror/snapshot/SetImageStateRequest.cc new file mode 100644 index 000000000..9fcee0322 --- /dev/null +++ b/src/librbd/mirror/snapshot/SetImageStateRequest.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/SetImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/image/GetMetadataRequest.h" +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" + +#include <boost/algorithm/string/predicate.hpp> + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror_snapshot::SetImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +void SetImageStateRequest<I>::send() { + get_name(); +} + +template <typename I> +void SetImageStateRequest<I>::get_name() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::dir_get_name_start(&op, m_image_ctx->id); + + librados::AioCompletion *comp = create_rados_callback< + SetImageStateRequest<I>, + &SetImageStateRequest<I>::handle_get_name>(this); + m_bl.clear(); + int r = m_image_ctx->md_ctx.aio_operate(RBD_DIRECTORY, comp, &op, &m_bl); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void SetImageStateRequest<I>::handle_get_name(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + auto it = m_bl.cbegin(); + r = cls_client::dir_get_name_finish(&it, &m_image_state.name); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve image name: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ldout(cct, 15) << "name=" << m_image_state.name << dendl; + + get_snap_limit(); +} + +template <typename I> +void SetImageStateRequest<I>::get_snap_limit() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectReadOperation op; + cls_client::snapshot_get_limit_start(&op); + + librados::AioCompletion *comp = create_rados_callback< + SetImageStateRequest<I>, + &SetImageStateRequest<I>::handle_get_snap_limit>(this); + m_bl.clear(); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, comp, &op, + &m_bl); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void SetImageStateRequest<I>::handle_get_snap_limit(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == 0) { + auto it = m_bl.cbegin(); + r = cls_client::snapshot_get_limit_finish(&it, &m_image_state.snap_limit); + } + + if (r < 0) { + lderr(cct) << "failed to retrieve snapshot limit: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + ldout(cct, 15) << "snap_limit=" << m_image_state.snap_limit << dendl; + + get_metadata(); +} + +template <typename I> +void SetImageStateRequest<I>::get_metadata() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + SetImageStateRequest<I>, + &SetImageStateRequest<I>::handle_get_metadata>(this); + auto req = image::GetMetadataRequest<I>::create( + m_image_ctx->md_ctx, m_image_ctx->header_oid, true, "", "", 0, + &m_image_state.metadata, ctx); + req->send(); +} + +template <typename I> +void SetImageStateRequest<I>::handle_get_metadata(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to retrieve metadata: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + + m_image_state.features = + m_image_ctx->features & ~RBD_FEATURES_IMPLICIT_ENABLE; + + for (auto &[snap_id, snap_info] : m_image_ctx->snap_info) { + auto type = cls::rbd::get_snap_namespace_type(snap_info.snap_namespace); + if (type != cls::rbd::SNAPSHOT_NAMESPACE_TYPE_USER) { + // only replicate user snapshots -- trash snapshots will be + // replicated by an implicit delete if required + continue; + } + m_image_state.snapshots[snap_id] = {snap_info.snap_namespace, + snap_info.name, + snap_info.protection_status}; + } + } + + write_image_state(); +} + +template <typename I> +void SetImageStateRequest<I>::write_image_state() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + SetImageStateRequest<I>, + &SetImageStateRequest<I>::handle_write_image_state>(this); + + auto req = WriteImageStateRequest<I>::create(m_image_ctx, m_snap_id, + m_image_state, ctx); + req->send(); +} + +template <typename I> +void SetImageStateRequest<I>::handle_write_image_state(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write image state: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + update_primary_snapshot(); +} + +template <typename I> +void SetImageStateRequest<I>::update_primary_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_snapshot_set_copy_progress( + &op, m_snap_id, true, 0); + + auto aio_comp = create_rados_callback< + SetImageStateRequest<I>, + &SetImageStateRequest<I>::handle_update_primary_snapshot>(this); + int r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void SetImageStateRequest<I>::handle_update_primary_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to update primary snapshot: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void SetImageStateRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::SetImageStateRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/SetImageStateRequest.h b/src/librbd/mirror/snapshot/SetImageStateRequest.h new file mode 100644 index 000000000..fd2815494 --- /dev/null +++ b/src/librbd/mirror/snapshot/SetImageStateRequest.h @@ -0,0 +1,96 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H + +#include "librbd/mirror/snapshot/Types.h" + +#include <map> +#include <string> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class SetImageStateRequest { +public: + static SetImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) { + return new SetImageStateRequest(image_ctx, snap_id, on_finish); + } + + SetImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * GET_NAME + * | + * v + * GET_SNAP_LIMIT + * | + * v + * GET_METADATA + * | + * v + * WRITE_IMAGE_STATE + * | + * v + * UPDATE_PRIMARY_SNAPSHOT + * | + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + Context *m_on_finish; + + ImageState m_image_state; + + bufferlist m_bl; + bufferlist m_state_bl; + + void get_name(); + void handle_get_name(int r); + + void get_snap_limit(); + void handle_get_snap_limit(int r); + + void get_metadata(); + void handle_get_metadata(int r); + + void write_image_state(); + void handle_write_image_state(int r); + + void update_primary_snapshot(); + void handle_update_primary_snapshot(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::SetImageStateRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_SET_IMAGE_STATE_REQUEST_H diff --git a/src/librbd/mirror/snapshot/Types.cc b/src/librbd/mirror/snapshot/Types.cc new file mode 100644 index 000000000..866b4c3e2 --- /dev/null +++ b/src/librbd/mirror/snapshot/Types.cc @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/Formatter.h" +#include "include/encoding.h" +#include "include/stringify.h" +#include "librbd/mirror/snapshot/Types.h" + +namespace librbd { +namespace mirror { +namespace snapshot { + +void ImageStateHeader::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(object_count, bl); + ENCODE_FINISH(bl); +} + +void ImageStateHeader::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(object_count, bl); + DECODE_FINISH(bl); +} + +void SnapState::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(snap_namespace, bl); + encode(name, bl); + encode(protection_status, bl); + ENCODE_FINISH(bl); +} + +void SnapState::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(snap_namespace, bl); + decode(name, bl); + decode(protection_status, bl); + DECODE_FINISH(bl); +} + +void SnapState::dump(Formatter *f) const { + f->open_object_section("namespace"); + snap_namespace.dump(f); + f->close_section(); + f->dump_string("name", name); + f->dump_unsigned("protection_status", protection_status); +} + +std::ostream& operator<<(std::ostream& os, const SnapState& snap_state) { + os << "[" + << "namespace=" << snap_state.snap_namespace << ", " + << "name=" << snap_state.name << ", " + << "protection=" << static_cast<int>(snap_state.protection_status) + << "]"; + return os; +} + +void ImageState::encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(name, bl); + encode(features, bl); + encode(snap_limit, bl); + encode(snapshots, bl); + encode(metadata, bl); + ENCODE_FINISH(bl); +} + +void ImageState::decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(name, bl); + decode(features, bl); + decode(snap_limit, bl); + decode(snapshots, bl); + decode(metadata, bl); + DECODE_FINISH(bl); +} + +void ImageState::dump(Formatter *f) const { + f->dump_string("name", name); + f->dump_unsigned("features", features); + f->dump_unsigned("snap_limit", snap_limit); + f->open_array_section("snapshots"); + for (auto &[id, snap_state] : snapshots) { + f->open_object_section(stringify(id).c_str()); + snap_state.dump(f); + f->close_section(); // snap_state + } + f->close_section(); // snapshots + f->open_object_section("metadata"); + for (auto &it : metadata) { + f->dump_stream(it.first.c_str()) << it.second; + } + f->close_section(); // metadata +} + +std::ostream& operator<<(std::ostream& os, const ImageState& image_state) { + os << "[" + << "name=" << image_state.name << ", " + << "features=" << image_state.features << ", " + << "snap_limit=" << image_state.snap_limit << ", " + << "snaps=" << image_state.snapshots << ", " + << "metadata_count=" << image_state.metadata.size() + << "]"; + return os; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd diff --git a/src/librbd/mirror/snapshot/Types.h b/src/librbd/mirror/snapshot/Types.h new file mode 100644 index 000000000..79947a5f8 --- /dev/null +++ b/src/librbd/mirror/snapshot/Types.h @@ -0,0 +1,122 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H + +#include "cls/rbd/cls_rbd_types.h" +#include "include/buffer.h" +#include "include/types.h" + +#include <map> +#include <string> + +namespace librbd { +namespace mirror { +namespace snapshot { + +enum CreatePrimaryFlags { + CREATE_PRIMARY_FLAG_IGNORE_EMPTY_PEERS = (1 << 0), + CREATE_PRIMARY_FLAG_DEMOTED = (1 << 1), + CREATE_PRIMARY_FLAG_FORCE = (1 << 2) +}; + +struct ImageStateHeader { + uint32_t object_count = 0; + + ImageStateHeader() { + } + ImageStateHeader(uint32_t object_count) : object_count(object_count) { + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); +}; + +WRITE_CLASS_ENCODER(ImageStateHeader); + +struct SnapState { + cls::rbd::SnapshotNamespace snap_namespace; + std::string name; + uint8_t protection_status = 0; + + SnapState() { + } + SnapState(const cls::rbd::SnapshotNamespace &snap_namespace, + const std::string &name, uint8_t protection_status) + : snap_namespace(snap_namespace), name(name), + protection_status(protection_status) { + } + + bool operator==(const SnapState& rhs) const { + return snap_namespace == rhs.snap_namespace && + name == rhs.name && protection_status == rhs.protection_status; + } + + bool operator<(const SnapState& rhs) const { + if (snap_namespace != rhs.snap_namespace) { + return snap_namespace < rhs.snap_namespace; + } + if (name != rhs.name) { + return name < rhs.name; + } + return protection_status < rhs.protection_status; + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); + void dump(Formatter *f) const; +}; + +std::ostream& operator<<(std::ostream& os, const SnapState& snap_state); + +WRITE_CLASS_ENCODER(SnapState); + +struct ImageState { + std::string name; + uint64_t features = 0; + uint64_t snap_limit = 0; + std::map<uint64_t, SnapState> snapshots; + std::map<std::string, bufferlist> metadata; + + ImageState() { + } + ImageState(const std::string &name, uint64_t features, uint64_t snap_limit, + const std::map<uint64_t, SnapState> &snapshots, + const std::map<std::string, bufferlist> &metadata) + : name(name), features(features), snap_limit(snap_limit), + snapshots(snapshots), metadata(metadata) { + } + + bool operator==(const ImageState& rhs) const { + return name == rhs.name && features == rhs.features && + snap_limit == rhs.snap_limit && snapshots == rhs.snapshots; + } + + bool operator<(const ImageState& rhs) const { + if (name != rhs.name) { + return name < rhs.name; + } + if (features != rhs.features) { + return features < rhs.features; + } + if (snap_limit != rhs.snap_limit) { + return snap_limit < rhs.snap_limit; + } + return snapshots < rhs.snapshots; + } + + void encode(bufferlist &bl) const; + void decode(bufferlist::const_iterator &it); + void dump(Formatter *f) const; +}; + +std::ostream& operator<<(std::ostream& os, const ImageState& image_state); + +WRITE_CLASS_ENCODER(ImageState); + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_TYPES_H diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc new file mode 100644 index 000000000..6e1884249 --- /dev/null +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.cc @@ -0,0 +1,235 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::UnlinkPeerRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +void UnlinkPeerRequest<I>::send() { + if (!m_image_ctx->state->is_refresh_required()) { + unlink_peer(); + return; + } + + refresh_image(); +} + +template <typename I> +void UnlinkPeerRequest<I>::refresh_image() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest<I>, &UnlinkPeerRequest<I>::handle_refresh_image>(this); + m_image_ctx->state->refresh(ctx); +} + +template <typename I> +void UnlinkPeerRequest<I>::handle_refresh_image(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to refresh image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + unlink_peer(); +} + +template <typename I> +void UnlinkPeerRequest<I>::unlink_peer() { + CephContext *cct = m_image_ctx->cct; + + m_image_ctx->image_lock.lock_shared(); + int r = -ENOENT; + cls::rbd::MirrorSnapshotNamespace* mirror_ns = nullptr; + m_newer_mirror_snapshots = false; + for (auto snap_it = m_image_ctx->snap_info.find(m_snap_id); + snap_it != m_image_ctx->snap_info.end(); ++snap_it) { + if (snap_it->first == m_snap_id) { + r = 0; + mirror_ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &snap_it->second.snap_namespace); + } else if (boost::get<cls::rbd::MirrorSnapshotNamespace>( + &snap_it->second.snap_namespace) != nullptr) { + ldout(cct, 15) << "located newer mirror snapshot" << dendl; + m_newer_mirror_snapshots = true; + break; + } + } + + if (r == -ENOENT) { + ldout(cct, 15) << "missing snapshot: snap_id=" << m_snap_id << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(r); + return; + } + + if (mirror_ns == nullptr) { + lderr(cct) << "not mirror snapshot (snap_id=" << m_snap_id << ")" << dendl; + m_image_ctx->image_lock.unlock_shared(); + finish(-EINVAL); + return; + } + + // if there is or will be no more peers in the mirror snapshot and we have + // a more recent mirror snapshot, remove the older one + if ((mirror_ns->mirror_peer_uuids.count(m_mirror_peer_uuid) == 0) || + (mirror_ns->mirror_peer_uuids.size() <= 1U && m_newer_mirror_snapshots)) { + m_image_ctx->image_lock.unlock_shared(); + remove_snapshot(); + return; + } + m_image_ctx->image_lock.unlock_shared(); + + ldout(cct, 15) << "snap_id=" << m_snap_id << ", " + << "mirror_peer_uuid=" << m_mirror_peer_uuid << dendl; + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_snapshot_unlink_peer(&op, m_snap_id, + m_mirror_peer_uuid); + auto aio_comp = create_rados_callback< + UnlinkPeerRequest<I>, &UnlinkPeerRequest<I>::handle_unlink_peer>(this); + r = m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void UnlinkPeerRequest<I>::handle_unlink_peer(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == -ERESTART || r == -ENOENT) { + refresh_image(); + return; + } + + if (r < 0) { + lderr(cct) << "failed to unlink peer: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + notify_update(); +} + +template <typename I> +void UnlinkPeerRequest<I>::notify_update() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + auto ctx = create_context_callback< + UnlinkPeerRequest<I>, &UnlinkPeerRequest<I>::handle_notify_update>(this); + m_image_ctx->notify_update(ctx); +} + +template <typename I> +void UnlinkPeerRequest<I>::handle_notify_update(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r == -ENOENT || r == -ETIMEDOUT) { + // non-fatel errors + lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; + } else if (r < 0) { + lderr(cct) << "failed to notify update: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + refresh_image(); +} + +template <typename I> +void UnlinkPeerRequest<I>::remove_snapshot() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << dendl; + + cls::rbd::SnapshotNamespace snap_namespace; + std::string snap_name; + int r = 0; + { + std::shared_lock image_locker{m_image_ctx->image_lock}; + + auto snap_info = m_image_ctx->get_snap_info(m_snap_id); + if (!snap_info) { + r = -ENOENT; + } else { + snap_namespace = snap_info->snap_namespace; + snap_name = snap_info->name; + } + } + + if (r == -ENOENT) { + ldout(cct, 15) << "failed to locate snapshot " << m_snap_id << dendl; + finish(0); + return; + } + + auto info = boost::get<cls::rbd::MirrorSnapshotNamespace>( + snap_namespace); + + info.mirror_peer_uuids.erase(m_mirror_peer_uuid); + if (!info.mirror_peer_uuids.empty() || !m_newer_mirror_snapshots) { + ldout(cct, 15) << "skipping removal of snapshot: " + << "snap_id=" << m_snap_id << ": " + << "mirror_peer_uuid=" << m_mirror_peer_uuid << ", " + << "mirror_peer_uuids=" << info.mirror_peer_uuids << dendl; + finish(0); + return; + } + + auto ctx = create_context_callback< + UnlinkPeerRequest<I>, &UnlinkPeerRequest<I>::handle_remove_snapshot>(this); + m_image_ctx->operations->snap_remove(snap_namespace, snap_name, ctx); +} + +template <typename I> +void UnlinkPeerRequest<I>::handle_remove_snapshot(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + lderr(cct) << "failed to remove snapshot: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void UnlinkPeerRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + auto on_finish = m_on_finish; + delete this; + on_finish->complete(r); +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::UnlinkPeerRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/UnlinkPeerRequest.h b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h new file mode 100644 index 000000000..9ef47269d --- /dev/null +++ b/src/librbd/mirror/snapshot/UnlinkPeerRequest.h @@ -0,0 +1,95 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H + +#include "include/buffer.h" + +#include <string> +#include <set> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class UnlinkPeerRequest { +public: + static UnlinkPeerRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + const std::string &mirror_peer_uuid, + Context *on_finish) { + return new UnlinkPeerRequest(image_ctx, snap_id, mirror_peer_uuid, + on_finish); + } + + UnlinkPeerRequest(ImageCtxT *image_ctx, uint64_t snap_id, + const std::string &mirror_peer_uuid, Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), + m_mirror_peer_uuid(mirror_peer_uuid), m_on_finish(on_finish) { + } + + void send(); + +private: + /* + * @verbatim + * + * <start> + * | + * v + * REFRESH_IMAGE <--------------------------\ + * | ^ (not found | + * | * or last) | + * | * | + * |\---------------> UNLINK_PEER --> NOTIFY_UPDATE + * | (not last peer or + * | no newer mirror + * | snap exists) + * | + * |\---------------> REMOVE_SNAPSHOT + * | (last peer and | + * | newer mirror | + * | snap exists) | + * | | + * |(peer not found) | + * v | + * <finish> <---------------/ + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + std::string m_mirror_peer_uuid; + Context *m_on_finish; + + bool m_newer_mirror_snapshots = false; + + void refresh_image(); + void handle_refresh_image(int r); + + void unlink_peer(); + void handle_unlink_peer(int r); + + void notify_update(); + void handle_notify_update(int r); + + void remove_snapshot(); + void handle_remove_snapshot(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::UnlinkPeerRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_UNLINK_PEER_REQUEST_H diff --git a/src/librbd/mirror/snapshot/Utils.cc b/src/librbd/mirror/snapshot/Utils.cc new file mode 100644 index 000000000..ecf884b54 --- /dev/null +++ b/src/librbd/mirror/snapshot/Utils.cc @@ -0,0 +1,186 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/dout.h" +#include "common/errno.h" +#include "include/stringify.h" +#include "librbd/ImageCtx.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::util: " \ + << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { +namespace util { + +namespace { + +const std::string IMAGE_STATE_OBJECT_PREFIX = "rbd_mirror_snapshot."; + +bool get_rollback_snap_id( + std::map<librados::snap_t, SnapInfo>::reverse_iterator it, + std::map<librados::snap_t, SnapInfo>::reverse_iterator end, + uint64_t *rollback_snap_id) { + + for (; it != end; it++) { + auto mirror_ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &it->second.snap_namespace); + if (mirror_ns->state != cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + break; + } + if (mirror_ns->complete) { + break; + } + } + + if (it != end) { + *rollback_snap_id = it->first; + return true; + } + + return false; +} + +} // anonymous namespace + +std::string get_image_meta_key(const std::string& mirror_uuid) { + return ".rbd_mirror." + mirror_uuid; +} + +template <typename I> +bool can_create_primary_snapshot(I *image_ctx, bool demoted, bool force, + bool* requires_orphan, + uint64_t *rollback_snap_id) { + CephContext *cct = image_ctx->cct; + + if (requires_orphan != nullptr) { + *requires_orphan = false; + } + if (rollback_snap_id) { + *rollback_snap_id = CEPH_NOSNAP; + } + + std::shared_lock image_locker{image_ctx->image_lock}; + + for (auto it = image_ctx->snap_info.rbegin(); + it != image_ctx->snap_info.rend(); it++) { + auto mirror_ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &it->second.snap_namespace); + if (mirror_ns == nullptr) { + continue; + } + ldout(cct, 20) << "previous snapshot snap_id=" << it->first << " " + << *mirror_ns << dendl; + if (mirror_ns->is_demoted() && !force) { + lderr(cct) << "trying to create primary snapshot without force " + << "when previous primary snapshot is demoted" + << dendl; + return false; + } + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + if (!force) { + lderr(cct) << "trying to create primary snapshot without force " + << "when previous snapshot is non-primary" + << dendl; + return false; + } + if (demoted) { + lderr(cct) << "trying to create primary demoted snapshot " + << "when previous snapshot is non-primary" + << dendl; + return false; + } + + if (requires_orphan != nullptr) { + *requires_orphan = !mirror_ns->is_demoted(); + } + if (!mirror_ns->complete) { + ldout(cct, 20) << "needs rollback" << dendl; + if (!rollback_snap_id) { + lderr(cct) << "trying to create primary snapshot " + << "when previous non-primary snapshot is not copied yet" + << dendl; + return false; + } + if (!get_rollback_snap_id(++it, image_ctx->snap_info.rend(), + rollback_snap_id)) { + lderr(cct) << "cannot rollback" << dendl; + return false; + } + ldout(cct, 20) << "rollback_snap_id=" << *rollback_snap_id << dendl; + } + return true; + } + + return true; + } + + ldout(cct, 20) << "no previous mirror snapshots found" << dendl; + return true; +} + +template <typename I> +bool can_create_non_primary_snapshot(I *image_ctx) { + CephContext *cct = image_ctx->cct; + + std::shared_lock image_locker{image_ctx->image_lock}; + + for (auto it = image_ctx->snap_info.rbegin(); + it != image_ctx->snap_info.rend(); it++) { + auto mirror_ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &it->second.snap_namespace); + if (mirror_ns != nullptr) { + ldout(cct, 20) << "previous mirror snapshot snap_id=" << it->first << " " + << *mirror_ns << dendl; + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY) { + if (!mirror_ns->complete) { + lderr(cct) << "trying to create non-primary snapshot " + << "when previous non-primary snapshot is not copied yet" + << dendl; + return false; + } + return true; + } + + if (mirror_ns->state == cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + lderr(cct) << "trying to create non-primary snapshot " + << "when previous primary snapshot is not in demoted state" + << dendl; + return false; + } + return true; + } + } + + ldout(cct, 20) << "no previous mirror snapshots found" << dendl; + return true; +} + +template <typename I> +std::string image_state_object_name(I *image_ctx, uint64_t snap_id, + uint64_t index) { + return IMAGE_STATE_OBJECT_PREFIX + image_ctx->id + "." + + stringify(snap_id) + "." + stringify(index); +} + +} // namespace util +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template bool librbd::mirror::snapshot::util::can_create_primary_snapshot( + librbd::ImageCtx *image_ctx, bool demoted, bool force, + bool* requires_orphan, uint64_t *rollback_snap_id); + +template bool librbd::mirror::snapshot::util::can_create_non_primary_snapshot( + librbd::ImageCtx *image_ctx); + +template std::string librbd::mirror::snapshot::util::image_state_object_name( + librbd::ImageCtx *image_ctx, uint64_t snap_id, uint64_t index); diff --git a/src/librbd/mirror/snapshot/Utils.h b/src/librbd/mirror/snapshot/Utils.h new file mode 100644 index 000000000..127ec5865 --- /dev/null +++ b/src/librbd/mirror/snapshot/Utils.h @@ -0,0 +1,38 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H + +#include "include/int_types.h" +#include "include/stringify.h" +#include <string> + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { +namespace util { + +std::string get_image_meta_key(const std::string& mirror_uuid); + +template <typename ImageCtxT = librbd::ImageCtx> +bool can_create_primary_snapshot(ImageCtxT *image_ctx, bool demoted, bool force, + bool* requires_orphan, + uint64_t *rollback_snap_id); + +template <typename ImageCtxT = librbd::ImageCtx> +bool can_create_non_primary_snapshot(ImageCtxT *image_ctx); + +template <typename ImageCtxT = librbd::ImageCtx> +std::string image_state_object_name(ImageCtxT *image_ctx, uint64_t snap_id, + uint64_t index); + +} // namespace util +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_UTILS_H diff --git a/src/librbd/mirror/snapshot/WriteImageStateRequest.cc b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc new file mode 100644 index 000000000..c79dd7e2c --- /dev/null +++ b/src/librbd/mirror/snapshot/WriteImageStateRequest.cc @@ -0,0 +1,120 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/mirror/snapshot/WriteImageStateRequest.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/Utils.h" + +#define dout_subsys ceph_subsys_rbd + +#undef dout_prefix +#define dout_prefix *_dout << "librbd::mirror::snapshot::WriteImageStateRequest: " \ + << this << " " << __func__ << ": " + +namespace librbd { +namespace mirror { +namespace snapshot { + +namespace { + +static size_t header_length() { + bufferlist bl; + ImageStateHeader header; + + using ceph::encode; + encode(header, bl); + + return bl.length(); +} + +} +using librbd::util::create_rados_callback; + +template <typename I> +WriteImageStateRequest<I>::WriteImageStateRequest(I *image_ctx, + uint64_t snap_id, + const ImageState &image_state, + Context *on_finish) + : m_image_ctx(image_ctx), m_snap_id(snap_id), m_image_state(image_state), + m_on_finish(on_finish), m_object_size( + 1 << image_ctx->config.template get_val<uint64_t>("rbd_default_order")) { + bufferlist bl; + encode(m_image_state, bl); + + m_object_count = 1 + (header_length() + bl.length()) / m_object_size; + ImageStateHeader header(m_object_count); + + encode(header, m_bl); + m_bl.claim_append(bl); +} + +template <typename I> +void WriteImageStateRequest<I>::send() { + write_object(); +} + +template <typename I> +void WriteImageStateRequest<I>::write_object() { + CephContext *cct = m_image_ctx->cct; + ceph_assert(m_object_count > 0); + + m_object_count--; + + auto oid = util::image_state_object_name(m_image_ctx, m_snap_id, + m_object_count); + ldout(cct, 15) << oid << dendl; + + size_t off = m_object_count * m_object_size; + size_t len = std::min(m_bl.length() - off, m_object_size); + bufferlist bl; + bl.substr_of(m_bl, off, len); + + librados::ObjectWriteOperation op; + op.write_full(bl); + + librados::AioCompletion *comp = create_rados_callback< + WriteImageStateRequest<I>, + &WriteImageStateRequest<I>::handle_write_object>(this); + int r = m_image_ctx->md_ctx.aio_operate(oid, comp, &op); + ceph_assert(r == 0); + comp->release(); +} + +template <typename I> +void WriteImageStateRequest<I>::handle_write_object(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + if (r < 0) { + lderr(cct) << "failed to write object: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_object_count == 0) { + finish(0); + return; + } + + write_object(); +} + +template <typename I> +void WriteImageStateRequest<I>::finish(int r) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +template class librbd::mirror::snapshot::WriteImageStateRequest<librbd::ImageCtx>; diff --git a/src/librbd/mirror/snapshot/WriteImageStateRequest.h b/src/librbd/mirror/snapshot/WriteImageStateRequest.h new file mode 100644 index 000000000..d2c4a7f80 --- /dev/null +++ b/src/librbd/mirror/snapshot/WriteImageStateRequest.h @@ -0,0 +1,73 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H +#define CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H + +#include "librbd/mirror/snapshot/Types.h" + +#include <map> +#include <string> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename ImageCtxT = librbd::ImageCtx> +class WriteImageStateRequest { +public: + static WriteImageStateRequest *create(ImageCtxT *image_ctx, uint64_t snap_id, + const ImageState &image_state, + Context *on_finish) { + return new WriteImageStateRequest(image_ctx, snap_id, image_state, + on_finish); + } + + WriteImageStateRequest(ImageCtxT *image_ctx, uint64_t snap_id, + const ImageState &image_state, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * WRITE_OBJECT (repeat for + * | every object) + * v + * <finish> + * + * @endverbatim + */ + + ImageCtxT *m_image_ctx; + uint64_t m_snap_id; + ImageState m_image_state; + Context *m_on_finish; + + bufferlist m_bl; + + const size_t m_object_size; + size_t m_object_count = 0; + + void write_object(); + void handle_write_object(int r); + + void finish(int r); +}; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +extern template class librbd::mirror::snapshot::WriteImageStateRequest<librbd::ImageCtx>; + +#endif // CEPH_LIBRBD_MIRROR_SNAPSHOT_WRITE_IMAGE_STATE_REQUEST_H |