diff options
Diffstat (limited to 'src/tools/rbd_mirror/image_replayer')
51 files changed, 10292 insertions, 0 deletions
diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc new file mode 100644 index 000000000..bda5b5f9b --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -0,0 +1,485 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/compat.h" +#include "BootstrapRequest.h" +#include "CreateImageRequest.h" +#include "OpenImageRequest.h" +#include "OpenLocalImageRequest.h" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "journal/Journaler.h" +#include "journal/Settings.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/internal.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/BaseRequest.h" +#include "tools/rbd_mirror/ImageSync.h" +#include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::" \ + << "BootstrapRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_context_callback; +using librbd::util::unique_lock_name; + +template <typename I> +BootstrapRequest<I>::BootstrapRequest( + Threads<I>* threads, + librados::IoCtx& local_io_ctx, + librados::IoCtx& remote_io_ctx, + InstanceWatcher<I>* instance_watcher, + const std::string& global_image_id, + const std::string& local_mirror_uuid, + const RemotePoolMeta& remote_pool_meta, + ::journal::CacheManagerHandler* cache_manager_handler, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + StateBuilder<I>** state_builder, + bool* do_resync, + Context* on_finish) + : CancelableRequest("rbd::mirror::image_replayer::BootstrapRequest", + reinterpret_cast<CephContext*>(local_io_ctx.cct()), + on_finish), + m_threads(threads), + m_local_io_ctx(local_io_ctx), + m_remote_io_ctx(remote_io_ctx), + m_instance_watcher(instance_watcher), + m_global_image_id(global_image_id), + m_local_mirror_uuid(local_mirror_uuid), + m_remote_pool_meta(remote_pool_meta), + m_cache_manager_handler(cache_manager_handler), + m_pool_meta_cache(pool_meta_cache), + m_progress_ctx(progress_ctx), + m_state_builder(state_builder), + m_do_resync(do_resync), + m_lock(ceph::make_mutex(unique_lock_name("BootstrapRequest::m_lock", + this))) { + dout(10) << dendl; +} + +template <typename I> +bool BootstrapRequest<I>::is_syncing() const { + std::lock_guard locker{m_lock}; + return (m_image_sync != nullptr); +} + +template <typename I> +void BootstrapRequest<I>::send() { + *m_do_resync = false; + + prepare_local_image(); +} + +template <typename I> +void BootstrapRequest<I>::cancel() { + dout(10) << dendl; + + std::lock_guard locker{m_lock}; + m_canceled = true; + + if (m_image_sync != nullptr) { + m_image_sync->cancel(); + } +} + +template <typename I> +std::string BootstrapRequest<I>::get_local_image_name() const { + std::unique_lock locker{m_lock}; + return m_local_image_name; +} + +template <typename I> +void BootstrapRequest<I>::prepare_local_image() { + dout(10) << dendl; + update_progress("PREPARE_LOCAL_IMAGE"); + + { + std::unique_lock locker{m_lock}; + m_local_image_name = m_global_image_id; + } + + ceph_assert(*m_state_builder == nullptr); + auto ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest<I>::handle_prepare_local_image>(this); + auto req = image_replayer::PrepareLocalImageRequest<I>::create( + m_local_io_ctx, m_global_image_id, &m_prepare_local_image_name, + m_state_builder, m_threads->work_queue, ctx); + req->send(); +} + +template <typename I> +void BootstrapRequest<I>::handle_prepare_local_image(int r) { + dout(10) << "r=" << r << dendl; + + ceph_assert(r < 0 || *m_state_builder != nullptr); + if (r == -ENOENT) { + dout(10) << "local image does not exist" << dendl; + } else if (r < 0) { + derr << "error preparing local image for replay: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + // image replayer will detect the name change (if any) at next + // status update + if (r >= 0 && !m_prepare_local_image_name.empty()) { + std::unique_lock locker{m_lock}; + m_local_image_name = m_prepare_local_image_name; + } + + prepare_remote_image(); +} + +template <typename I> +void BootstrapRequest<I>::prepare_remote_image() { + dout(10) << dendl; + update_progress("PREPARE_REMOTE_IMAGE"); + + Context *ctx = create_context_callback< + BootstrapRequest, &BootstrapRequest<I>::handle_prepare_remote_image>(this); + auto req = image_replayer::PrepareRemoteImageRequest<I>::create( + m_threads, m_local_io_ctx, m_remote_io_ctx, m_global_image_id, + m_local_mirror_uuid, m_remote_pool_meta, m_cache_manager_handler, + m_state_builder, ctx); + req->send(); +} + +template <typename I> +void BootstrapRequest<I>::handle_prepare_remote_image(int r) { + dout(10) << "r=" << r << dendl; + + auto state_builder = *m_state_builder; + ceph_assert(state_builder == nullptr || + !state_builder->remote_mirror_uuid.empty()); + + if (state_builder != nullptr && state_builder->is_local_primary()) { + dout(5) << "local image is primary" << dendl; + finish(-ENOMSG); + return; + } else if (r == -ENOENT || state_builder == nullptr) { + dout(10) << "remote image does not exist"; + if (state_builder != nullptr) { + *_dout << ": " + << "local_image_id=" << state_builder->local_image_id << ", " + << "remote_image_id=" << state_builder->remote_image_id << ", " + << "is_linked=" << state_builder->is_linked(); + } + *_dout << dendl; + + // TODO need to support multiple remote images + if (state_builder != nullptr && + state_builder->remote_image_id.empty() && + (state_builder->local_image_id.empty() || + state_builder->is_linked())) { + // both images doesn't exist or local image exists and is non-primary + // and linked to the missing remote image + finish(-ENOLINK); + } else { + finish(-ENOENT); + } + return; + } else if (r < 0) { + derr << "error preparing remote image for replay: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (!state_builder->is_remote_primary()) { + ceph_assert(!state_builder->remote_image_id.empty()); + if (state_builder->local_image_id.empty()) { + dout(10) << "local image does not exist and remote image is not primary" + << dendl; + finish(-EREMOTEIO); + return; + } else if (!state_builder->is_linked()) { + dout(10) << "local image is unlinked and remote image is not primary" + << dendl; + finish(-EREMOTEIO); + return; + } + // if the local image is linked to the remote image, we ignore that + // the remote image is not primary so that we can replay demotion + } + + open_remote_image(); +} + +template <typename I> +void BootstrapRequest<I>::open_remote_image() { + ceph_assert(*m_state_builder != nullptr); + auto remote_image_id = (*m_state_builder)->remote_image_id; + dout(15) << "remote_image_id=" << remote_image_id << dendl; + + update_progress("OPEN_REMOTE_IMAGE"); + + auto ctx = create_context_callback< + BootstrapRequest<I>, + &BootstrapRequest<I>::handle_open_remote_image>(this); + ceph_assert(*m_state_builder != nullptr); + OpenImageRequest<I> *request = OpenImageRequest<I>::create( + m_remote_io_ctx, &(*m_state_builder)->remote_image_ctx, remote_image_id, + false, ctx); + request->send(); +} + +template <typename I> +void BootstrapRequest<I>::handle_open_remote_image(int r) { + dout(15) << "r=" << r << dendl; + + ceph_assert(*m_state_builder != nullptr); + if (r < 0) { + derr << "failed to open remote image: " << cpp_strerror(r) << dendl; + ceph_assert((*m_state_builder)->remote_image_ctx == nullptr); + finish(r); + return; + } + + if ((*m_state_builder)->local_image_id.empty()) { + create_local_image(); + return; + } + + open_local_image(); +} + +template <typename I> +void BootstrapRequest<I>::open_local_image() { + ceph_assert(*m_state_builder != nullptr); + auto local_image_id = (*m_state_builder)->local_image_id; + + dout(15) << "local_image_id=" << local_image_id << dendl; + + update_progress("OPEN_LOCAL_IMAGE"); + + Context *ctx = create_context_callback< + BootstrapRequest<I>, &BootstrapRequest<I>::handle_open_local_image>( + this); + OpenLocalImageRequest<I> *request = OpenLocalImageRequest<I>::create( + m_local_io_ctx, &(*m_state_builder)->local_image_ctx, local_image_id, + m_threads->work_queue, ctx); + request->send(); +} + +template <typename I> +void BootstrapRequest<I>::handle_open_local_image(int r) { + dout(15) << "r=" << r << dendl; + + ceph_assert(*m_state_builder != nullptr); + auto local_image_ctx = (*m_state_builder)->local_image_ctx; + ceph_assert((r >= 0 && local_image_ctx != nullptr) || + (r < 0 && local_image_ctx == nullptr)); + + if (r == -ENOENT) { + dout(10) << "local image missing" << dendl; + create_local_image(); + return; + } else if (r == -EREMOTEIO) { + dout(10) << "local image is primary -- skipping image replay" << dendl; + m_ret_val = r; + close_remote_image(); + return; + } else if (r < 0) { + derr << "failed to open local image: " << cpp_strerror(r) << dendl; + m_ret_val = r; + close_remote_image(); + return; + } + + prepare_replay(); +} + +template <typename I> +void BootstrapRequest<I>::prepare_replay() { + dout(10) << dendl; + update_progress("PREPARE_REPLAY"); + + ceph_assert(*m_state_builder != nullptr); + auto ctx = create_context_callback< + BootstrapRequest<I>, &BootstrapRequest<I>::handle_prepare_replay>(this); + auto request = (*m_state_builder)->create_prepare_replay_request( + m_local_mirror_uuid, m_progress_ctx, m_do_resync, &m_syncing, ctx); + request->send(); +} + +template <typename I> +void BootstrapRequest<I>::handle_prepare_replay(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to prepare local replay: " << cpp_strerror(r) << dendl; + m_ret_val = r; + close_remote_image(); + return; + } else if (*m_do_resync) { + dout(10) << "local image resync requested" << dendl; + close_remote_image(); + return; + } else if ((*m_state_builder)->is_disconnected()) { + dout(10) << "client flagged disconnected -- skipping bootstrap" << dendl; + // The caller is expected to detect disconnect initializing remote journal. + m_ret_val = 0; + close_remote_image(); + return; + } else if (m_syncing) { + dout(10) << "local image still syncing to remote image" << dendl; + image_sync(); + return; + } + + close_remote_image(); +} + +template <typename I> +void BootstrapRequest<I>::create_local_image() { + dout(10) << dendl; + update_progress("CREATE_LOCAL_IMAGE"); + + ceph_assert(*m_state_builder != nullptr); + auto ctx = create_context_callback< + BootstrapRequest<I>, + &BootstrapRequest<I>::handle_create_local_image>(this); + auto request = (*m_state_builder)->create_local_image_request( + m_threads, m_local_io_ctx, m_global_image_id, m_pool_meta_cache, + m_progress_ctx, ctx); + request->send(); +} + +template <typename I> +void BootstrapRequest<I>::handle_create_local_image(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + if (r == -ENOENT) { + dout(10) << "parent image does not exist" << dendl; + } else { + derr << "failed to create local image: " << cpp_strerror(r) << dendl; + } + m_ret_val = r; + close_remote_image(); + return; + } + + open_local_image(); +} + +template <typename I> +void BootstrapRequest<I>::image_sync() { + std::unique_lock locker{m_lock}; + if (m_canceled) { + locker.unlock(); + + m_ret_val = -ECANCELED; + dout(10) << "request canceled" << dendl; + close_remote_image(); + return; + } + + dout(15) << dendl; + ceph_assert(m_image_sync == nullptr); + + auto state_builder = *m_state_builder; + auto sync_point_handler = state_builder->create_sync_point_handler(); + + Context *ctx = create_context_callback< + BootstrapRequest<I>, &BootstrapRequest<I>::handle_image_sync>(this); + m_image_sync = ImageSync<I>::create( + m_threads, state_builder->local_image_ctx, state_builder->remote_image_ctx, + m_local_mirror_uuid, sync_point_handler, m_instance_watcher, + m_progress_ctx, ctx); + m_image_sync->get(); + locker.unlock(); + + update_progress("IMAGE_SYNC"); + m_image_sync->send(); +} + +template <typename I> +void BootstrapRequest<I>::handle_image_sync(int r) { + dout(15) << "r=" << r << dendl; + + { + std::lock_guard locker{m_lock}; + m_image_sync->put(); + m_image_sync = nullptr; + + (*m_state_builder)->destroy_sync_point_handler(); + } + + if (r < 0) { + if (r == -ECANCELED) { + dout(10) << "request canceled" << dendl; + } else { + derr << "failed to sync remote image: " << cpp_strerror(r) << dendl; + } + m_ret_val = r; + } + + close_remote_image(); +} + +template <typename I> +void BootstrapRequest<I>::close_remote_image() { + if ((*m_state_builder)->replay_requires_remote_image()) { + finish(m_ret_val); + return; + } + + dout(15) << dendl; + + update_progress("CLOSE_REMOTE_IMAGE"); + + auto ctx = create_context_callback< + BootstrapRequest<I>, + &BootstrapRequest<I>::handle_close_remote_image>(this); + ceph_assert(*m_state_builder != nullptr); + (*m_state_builder)->close_remote_image(ctx); +} + +template <typename I> +void BootstrapRequest<I>::handle_close_remote_image(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "error encountered closing remote image: " << cpp_strerror(r) + << dendl; + } + + finish(m_ret_val); +} + +template <typename I> +void BootstrapRequest<I>::update_progress(const std::string &description) { + dout(15) << description << dendl; + + if (m_progress_ctx) { + m_progress_ctx->update_progress(description); + } +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::BootstrapRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h new file mode 100644 index 000000000..f5bb8dd8a --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -0,0 +1,181 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_BOOTSTRAP_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_BOOTSTRAP_REQUEST_H + +#include "include/int_types.h" +#include "include/rados/librados.hpp" +#include "common/ceph_mutex.h" +#include "common/Timer.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" +#include "tools/rbd_mirror/CancelableRequest.h" +#include "tools/rbd_mirror/Types.h" +#include <string> + +class Context; + +namespace journal { class CacheManagerHandler; } +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { + +class ProgressContext; + +template <typename> class ImageSync; +template <typename> class InstanceWatcher; +struct PoolMetaCache; +template <typename> struct Threads; + +namespace image_replayer { + +template <typename> class StateBuilder; + +template <typename ImageCtxT = librbd::ImageCtx> +class BootstrapRequest : public CancelableRequest { +public: + typedef rbd::mirror::ProgressContext ProgressContext; + + static BootstrapRequest* create( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + librados::IoCtx& remote_io_ctx, + InstanceWatcher<ImageCtxT>* instance_watcher, + const std::string& global_image_id, + const std::string& local_mirror_uuid, + const RemotePoolMeta& remote_pool_meta, + ::journal::CacheManagerHandler* cache_manager_handler, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>** state_builder, + bool* do_resync, + Context* on_finish) { + return new BootstrapRequest( + threads, local_io_ctx, remote_io_ctx, instance_watcher, global_image_id, + local_mirror_uuid, remote_pool_meta, cache_manager_handler, + pool_meta_cache, progress_ctx, state_builder, do_resync, on_finish); + } + + BootstrapRequest( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + librados::IoCtx& remote_io_ctx, + InstanceWatcher<ImageCtxT>* instance_watcher, + const std::string& global_image_id, + const std::string& local_mirror_uuid, + const RemotePoolMeta& remote_pool_meta, + ::journal::CacheManagerHandler* cache_manager_handler, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>** state_builder, + bool* do_resync, + Context* on_finish); + + bool is_syncing() const; + + void send() override; + void cancel() override; + + std::string get_local_image_name() const; + +private: + /** + * @verbatim + * + * <start> + * | + * v (error) + * PREPARE_LOCAL_IMAGE * * * * * * * * * * * * * * * * * * + * | * + * v (error) * + * PREPARE_REMOTE_IMAGE * * * * * * * * * * * * * * * * * * + * | * + * v (error) * + * OPEN_REMOTE_IMAGE * * * * * * * * * * * * * * * * * * * + * | * + * | * + * \----> CREATE_LOCAL_IMAGE * * * * * * * * * * * * * + * | | ^ * * + * | | . * * + * | v . (image DNE) * * + * \----> OPEN_LOCAL_IMAGE * * * * * * * * * * * * * * + * | * * + * | * * + * v * * + * PREPARE_REPLAY * * * * * * * * * * * * * * * + * | * * + * | * * + * v (skip if not needed) * * + * IMAGE_SYNC * * * * * * * * * * * * * * * * * + * | * * + * | * * + * /---------/ * * + * | * * + * v * * + * CLOSE_REMOTE_IMAGE < * * * * * * * * * * * * * * * * * + * | * + * v * + * <finish> < * * * * * * * * * * * * * * * * * * * * * * * + * + * @endverbatim + */ + Threads<ImageCtxT>* m_threads; + librados::IoCtx &m_local_io_ctx; + librados::IoCtx &m_remote_io_ctx; + InstanceWatcher<ImageCtxT> *m_instance_watcher; + std::string m_global_image_id; + std::string m_local_mirror_uuid; + RemotePoolMeta m_remote_pool_meta; + ::journal::CacheManagerHandler *m_cache_manager_handler; + PoolMetaCache* m_pool_meta_cache; + ProgressContext *m_progress_ctx; + StateBuilder<ImageCtxT>** m_state_builder; + bool *m_do_resync; + + mutable ceph::mutex m_lock; + bool m_canceled = false; + + int m_ret_val = 0; + + std::string m_local_image_name; + std::string m_prepare_local_image_name; + + bool m_syncing = false; + ImageSync<ImageCtxT> *m_image_sync = nullptr; + + void prepare_local_image(); + void handle_prepare_local_image(int r); + + void prepare_remote_image(); + void handle_prepare_remote_image(int r); + + void open_remote_image(); + void handle_open_remote_image(int r); + + void open_local_image(); + void handle_open_local_image(int r); + + void create_local_image(); + void handle_create_local_image(int r); + + void prepare_replay(); + void handle_prepare_replay(int r); + + void image_sync(); + void handle_image_sync(int r); + + void close_remote_image(); + void handle_close_remote_image(int r); + + void update_progress(const std::string &description); +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::BootstrapRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_BOOTSTRAP_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/CloseImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.cc new file mode 100644 index 000000000..872c8baa9 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.cc @@ -0,0 +1,62 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "CloseImageRequest.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Utils.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::CloseImageRequest: " \ + << this << " " << __func__ + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_context_callback; + +template <typename I> +CloseImageRequest<I>::CloseImageRequest(I **image_ctx, Context *on_finish) + : m_image_ctx(image_ctx), m_on_finish(on_finish) { +} + +template <typename I> +void CloseImageRequest<I>::send() { + close_image(); +} + +template <typename I> +void CloseImageRequest<I>::close_image() { + dout(20) << dendl; + + Context *ctx = create_context_callback< + CloseImageRequest<I>, &CloseImageRequest<I>::handle_close_image>(this); + (*m_image_ctx)->state->close(ctx); +} + +template <typename I> +void CloseImageRequest<I>::handle_close_image(int r) { + dout(20) << ": r=" << r << dendl; + + if (r < 0) { + derr << ": error encountered while closing image: " << cpp_strerror(r) + << dendl; + } + + *m_image_ctx = nullptr; + + m_on_finish->complete(0); + delete this; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::CloseImageRequest<librbd::ImageCtx>; + diff --git a/src/tools/rbd_mirror/image_replayer/CloseImageRequest.h b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.h new file mode 100644 index 000000000..02481369d --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/CloseImageRequest.h @@ -0,0 +1,56 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_CLOSE_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_CLOSE_IMAGE_REQUEST_H + +#include "include/int_types.h" +#include "librbd/ImageCtx.h" +#include <string> + +class Context; +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <typename ImageCtxT = librbd::ImageCtx> +class CloseImageRequest { +public: + static CloseImageRequest* create(ImageCtxT **image_ctx, Context *on_finish) { + return new CloseImageRequest(image_ctx, on_finish); + } + + CloseImageRequest(ImageCtxT **image_ctx, Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * CLOSE_IMAGE + * | + * v + * <finish> + * + * @endverbatim + */ + ImageCtxT **m_image_ctx; + Context *m_on_finish; + + void close_image(); + void handle_close_image(int r); +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::CloseImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_CLOSE_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc new file mode 100644 index 000000000..641bb03e8 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.cc @@ -0,0 +1,451 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "CreateImageRequest.h" +#include "CloseImageRequest.h" +#include "OpenImageRequest.h" +#include "common/debug.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/internal.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/image/CreateRequest.h" +#include "librbd/image/CloneRequest.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Types.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_sync/Utils.h" +#include <boost/algorithm/string/predicate.hpp> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::CreateImageRequest: " \ + << this << " " << __func__ << ": " + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <typename I> +CreateImageRequest<I>::CreateImageRequest( + Threads<I>* threads, + librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + I *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish) + : m_threads(threads), m_local_io_ctx(local_io_ctx), + m_global_image_id(global_image_id), + m_remote_mirror_uuid(remote_mirror_uuid), + m_local_image_name(local_image_name), m_local_image_id(local_image_id), + m_remote_image_ctx(remote_image_ctx), + m_pool_meta_cache(pool_meta_cache), + m_mirror_image_mode(mirror_image_mode), m_on_finish(on_finish) { +} + +template <typename I> +void CreateImageRequest<I>::send() { + int r = validate_parent(); + if (r < 0) { + error(r); + return; + } + + if (m_remote_parent_spec.pool_id == -1) { + create_image(); + } else { + get_parent_global_image_id(); + } +} + +template <typename I> +void CreateImageRequest<I>::create_image() { + dout(10) << dendl; + + using klass = CreateImageRequest<I>; + Context *ctx = create_context_callback< + klass, &klass::handle_create_image>(this); + + std::shared_lock image_locker{m_remote_image_ctx->image_lock}; + + auto& config{ + reinterpret_cast<CephContext*>(m_local_io_ctx.cct())->_conf}; + + librbd::ImageOptions image_options; + populate_image_options(&image_options); + + auto req = librbd::image::CreateRequest<I>::create( + config, m_local_io_ctx, m_local_image_name, m_local_image_id, + m_remote_image_ctx->size, image_options, 0U, m_mirror_image_mode, + m_global_image_id, m_remote_mirror_uuid, m_remote_image_ctx->op_work_queue, + ctx); + req->send(); +} + +template <typename I> +void CreateImageRequest<I>::handle_create_image(int r) { + dout(10) << "r=" << r << dendl; + if (r == -EBADF) { + dout(5) << "image id " << m_local_image_id << " already in-use" << dendl; + finish(r); + return; + } else if (r < 0) { + derr << "failed to create local image: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void CreateImageRequest<I>::get_parent_global_image_id() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_start(&op, + m_remote_parent_spec.image_id); + + librados::AioCompletion *aio_comp = create_rados_callback< + CreateImageRequest<I>, + &CreateImageRequest<I>::handle_get_parent_global_image_id>(this); + m_out_bl.clear(); + int r = m_remote_parent_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, + &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void CreateImageRequest<I>::handle_get_parent_global_image_id(int r) { + dout(10) << "r=" << r << dendl; + if (r == 0) { + cls::rbd::MirrorImage mirror_image; + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_get_finish(&iter, &mirror_image); + if (r == 0) { + m_parent_global_image_id = mirror_image.global_image_id; + dout(15) << "parent_global_image_id=" << m_parent_global_image_id + << dendl; + } + } + + if (r == -ENOENT) { + dout(10) << "parent image " << m_remote_parent_spec.image_id + << " not mirrored" << dendl; + finish(r); + return; + } else if (r < 0) { + derr << "failed to retrieve global image id for parent image " + << m_remote_parent_spec.image_id << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_local_parent_image_id(); +} + +template <typename I> +void CreateImageRequest<I>::get_local_parent_image_id() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_image_id_start( + &op, m_parent_global_image_id); + + librados::AioCompletion *aio_comp = create_rados_callback< + CreateImageRequest<I>, + &CreateImageRequest<I>::handle_get_local_parent_image_id>(this); + m_out_bl.clear(); + int r = m_local_parent_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, + &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void CreateImageRequest<I>::handle_get_local_parent_image_id(int r) { + dout(10) << "r=" << r << dendl; + + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_get_image_id_finish( + &iter, &m_local_parent_spec.image_id); + } + + if (r == -ENOENT) { + dout(10) << "parent image " << m_parent_global_image_id << " not " + << "registered locally" << dendl; + finish(r); + return; + } else if (r < 0) { + derr << "failed to retrieve local image id for parent image " + << m_parent_global_image_id << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + open_remote_parent_image(); +} + +template <typename I> +void CreateImageRequest<I>::open_remote_parent_image() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + CreateImageRequest<I>, + &CreateImageRequest<I>::handle_open_remote_parent_image>(this); + OpenImageRequest<I> *request = OpenImageRequest<I>::create( + m_remote_parent_io_ctx, &m_remote_parent_image_ctx, + m_remote_parent_spec.image_id, true, ctx); + request->send(); +} + +template <typename I> +void CreateImageRequest<I>::handle_open_remote_parent_image(int r) { + dout(10) << "r=" << r << dendl; + if (r < 0) { + derr << "failed to open remote parent image " << m_parent_pool_name << "/" + << m_remote_parent_spec.image_id << dendl; + finish(r); + return; + } + + clone_image(); +} + +template <typename I> +void CreateImageRequest<I>::clone_image() { + dout(10) << dendl; + + LocalPoolMeta local_parent_pool_meta; + int r = m_pool_meta_cache->get_local_pool_meta( + m_local_parent_io_ctx.get_id(), &local_parent_pool_meta); + if (r < 0) { + derr << "failed to retrieve local parent mirror uuid for pool " + << m_local_parent_io_ctx.get_id() << dendl; + m_ret_val = r; + close_remote_parent_image(); + return; + } + + // ensure no image sync snapshots for the local cluster exist in the + // remote image + bool found_parent_snap = false; + bool found_image_sync_snap = false; + std::string snap_name; + cls::rbd::SnapshotNamespace snap_namespace; + { + auto snap_prefix = image_sync::util::get_snapshot_name_prefix( + local_parent_pool_meta.mirror_uuid); + + std::shared_lock remote_image_locker(m_remote_parent_image_ctx->image_lock); + for (auto snap_info : m_remote_parent_image_ctx->snap_info) { + if (snap_info.first == m_remote_parent_spec.snap_id) { + found_parent_snap = true; + snap_name = snap_info.second.name; + snap_namespace = snap_info.second.snap_namespace; + } else if (boost::starts_with(snap_info.second.name, snap_prefix)) { + found_image_sync_snap = true; + } + } + } + + if (!found_parent_snap) { + dout(15) << "remote parent image snapshot not found" << dendl; + m_ret_val = -ENOENT; + close_remote_parent_image(); + return; + } else if (found_image_sync_snap) { + dout(15) << "parent image not synced to local cluster" << dendl; + m_ret_val = -ENOENT; + close_remote_parent_image(); + return; + } + + librbd::ImageOptions opts; + populate_image_options(&opts); + + auto& config{ + reinterpret_cast<CephContext*>(m_local_io_ctx.cct())->_conf}; + + using klass = CreateImageRequest<I>; + Context *ctx = create_context_callback< + klass, &klass::handle_clone_image>(this); + + librbd::image::CloneRequest<I> *req = librbd::image::CloneRequest<I>::create( + config, m_local_parent_io_ctx, m_local_parent_spec.image_id, snap_name, + snap_namespace, CEPH_NOSNAP, m_local_io_ctx, m_local_image_name, + m_local_image_id, opts, m_mirror_image_mode, m_global_image_id, + m_remote_mirror_uuid, m_remote_image_ctx->op_work_queue, ctx); + req->send(); +} + +template <typename I> +void CreateImageRequest<I>::handle_clone_image(int r) { + dout(10) << "r=" << r << dendl; + if (r == -EBADF) { + dout(5) << "image id " << m_local_image_id << " already in-use" << dendl; + m_ret_val = r; + } else if (r < 0) { + derr << "failed to clone image " << m_parent_pool_name << "/" + << m_remote_parent_spec.image_id << " to " + << m_local_image_name << dendl; + m_ret_val = r; + } + + close_remote_parent_image(); +} + +template <typename I> +void CreateImageRequest<I>::close_remote_parent_image() { + dout(10) << dendl; + Context *ctx = create_context_callback< + CreateImageRequest<I>, + &CreateImageRequest<I>::handle_close_remote_parent_image>(this); + CloseImageRequest<I> *request = CloseImageRequest<I>::create( + &m_remote_parent_image_ctx, ctx); + request->send(); +} + +template <typename I> +void CreateImageRequest<I>::handle_close_remote_parent_image(int r) { + dout(10) << "r=" << r << dendl; + if (r < 0) { + derr << "error encountered closing remote parent image: " + << cpp_strerror(r) << dendl; + } + + finish(m_ret_val); +} + +template <typename I> +void CreateImageRequest<I>::error(int r) { + dout(10) << "r=" << r << dendl; + + m_threads->work_queue->queue(create_context_callback< + CreateImageRequest<I>, &CreateImageRequest<I>::finish>(this), r); +} + +template <typename I> +void CreateImageRequest<I>::finish(int r) { + dout(10) << "r=" << r << dendl; + m_on_finish->complete(r); + delete this; +} + +template <typename I> +int CreateImageRequest<I>::validate_parent() { + std::shared_lock owner_locker{m_remote_image_ctx->owner_lock}; + std::shared_lock image_locker{m_remote_image_ctx->image_lock}; + + m_remote_parent_spec = m_remote_image_ctx->parent_md.spec; + + // scan all remote snapshots for a linked parent + for (auto &snap_info_pair : m_remote_image_ctx->snap_info) { + auto &parent_spec = snap_info_pair.second.parent.spec; + if (parent_spec.pool_id == -1) { + continue; + } else if (m_remote_parent_spec.pool_id == -1) { + m_remote_parent_spec = parent_spec; + continue; + } + + if (m_remote_parent_spec != parent_spec) { + derr << "remote image parent spec mismatch" << dendl; + return -EINVAL; + } + } + + if (m_remote_parent_spec.pool_id == -1) { + return 0; + } + + // map remote parent pool to local parent pool + int r = librbd::util::create_ioctx( + m_remote_image_ctx->md_ctx, "remote parent pool", + m_remote_parent_spec.pool_id, m_remote_parent_spec.pool_namespace, + &m_remote_parent_io_ctx); + if (r < 0) { + derr << "failed to open remote parent pool " << m_remote_parent_spec.pool_id + << ": " << cpp_strerror(r) << dendl; + return r; + } + + m_parent_pool_name = m_remote_parent_io_ctx.get_pool_name(); + + librados::Rados local_rados(m_local_io_ctx); + r = local_rados.ioctx_create(m_parent_pool_name.c_str(), + m_local_parent_io_ctx); + if (r < 0) { + derr << "failed to open local parent pool " << m_parent_pool_name << ": " + << cpp_strerror(r) << dendl; + return r; + } + m_local_parent_io_ctx.set_namespace(m_remote_parent_io_ctx.get_namespace()); + + return 0; +} + +template <typename I> +void CreateImageRequest<I>::populate_image_options( + librbd::ImageOptions* image_options) { + image_options->set(RBD_IMAGE_OPTION_FEATURES, + m_remote_image_ctx->features); + image_options->set(RBD_IMAGE_OPTION_ORDER, m_remote_image_ctx->order); + image_options->set(RBD_IMAGE_OPTION_STRIPE_UNIT, + m_remote_image_ctx->stripe_unit); + image_options->set(RBD_IMAGE_OPTION_STRIPE_COUNT, + m_remote_image_ctx->stripe_count); + + // Determine the data pool for the local image as follows: + // 1. If the local pool has a default data pool, use it. + // 2. If the remote image has a data pool different from its metadata pool and + // a pool with the same name exists locally, use it. + // 3. Don't set the data pool explicitly. + std::string data_pool; + librados::Rados local_rados(m_local_io_ctx); + auto default_data_pool = g_ceph_context->_conf.get_val<std::string>("rbd_default_data_pool"); + auto remote_md_pool = m_remote_image_ctx->md_ctx.get_pool_name(); + auto remote_data_pool = m_remote_image_ctx->data_ctx.get_pool_name(); + + if (default_data_pool != "") { + data_pool = default_data_pool; + } else if (remote_data_pool != remote_md_pool) { + if (local_rados.pool_lookup(remote_data_pool.c_str()) >= 0) { + data_pool = remote_data_pool; + } + } + + if (data_pool != "") { + image_options->set(RBD_IMAGE_OPTION_DATA_POOL, data_pool); + } + + if (m_remote_parent_spec.pool_id != -1) { + uint64_t clone_format = 1; + if (m_remote_image_ctx->test_op_features( + RBD_OPERATION_FEATURE_CLONE_CHILD)) { + clone_format = 2; + } + image_options->set(RBD_IMAGE_OPTION_CLONE_FORMAT, clone_format); + } +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::CreateImageRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h new file mode 100644 index 000000000..2ff7794e8 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/CreateImageRequest.h @@ -0,0 +1,144 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_CREATE_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_CREATE_IMAGE_REQUEST_H + +#include "include/int_types.h" +#include "include/types.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/Types.h" +#include <string> + +class Context; +namespace librbd { class ImageCtx; } +namespace librbd { class ImageOptions; } + +namespace rbd { +namespace mirror { + +class PoolMetaCache; +template <typename> struct Threads; + +namespace image_replayer { + +template <typename ImageCtxT = librbd::ImageCtx> +class CreateImageRequest { +public: + static CreateImageRequest *create( + Threads<ImageCtxT> *threads, + librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + ImageCtxT *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish) { + return new CreateImageRequest(threads, local_io_ctx, global_image_id, + remote_mirror_uuid, local_image_name, + local_image_id, remote_image_ctx, + pool_meta_cache, mirror_image_mode, + on_finish); + } + + CreateImageRequest( + Threads<ImageCtxT> *threads, librados::IoCtx &local_io_ctx, + const std::string &global_image_id, + const std::string &remote_mirror_uuid, + const std::string &local_image_name, + const std::string &local_image_id, + ImageCtxT *remote_image_ctx, + PoolMetaCache* pool_meta_cache, + cls::rbd::MirrorImageMode mirror_image_mode, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * | * + * | (non-clone) * + * |\------------> CREATE_IMAGE ---------------------\ * (error) + * | | * + * | (clone) | * + * \-------------> GET_PARENT_GLOBAL_IMAGE_ID * * * | * * * * + * | | * * + * v | * + * GET_LOCAL_PARENT_IMAGE_ID * * * * | * * * * + * | | * * + * v | * + * OPEN_REMOTE_PARENT * * * * * * * | * * * * + * | | * * + * v | * + * CLONE_IMAGE | * + * | | * + * v | * + * CLOSE_REMOTE_PARENT | * + * | v * + * \------------------------> <finish> < * * + * @endverbatim + */ + + Threads<ImageCtxT> *m_threads; + librados::IoCtx &m_local_io_ctx; + std::string m_global_image_id; + std::string m_remote_mirror_uuid; + std::string m_local_image_name; + std::string m_local_image_id; + ImageCtxT *m_remote_image_ctx; + PoolMetaCache* m_pool_meta_cache; + cls::rbd::MirrorImageMode m_mirror_image_mode; + Context *m_on_finish; + + librados::IoCtx m_remote_parent_io_ctx; + ImageCtxT *m_remote_parent_image_ctx = nullptr; + cls::rbd::ParentImageSpec m_remote_parent_spec; + + librados::IoCtx m_local_parent_io_ctx; + cls::rbd::ParentImageSpec m_local_parent_spec; + + bufferlist m_out_bl; + std::string m_parent_global_image_id; + std::string m_parent_pool_name; + int m_ret_val = 0; + + void create_image(); + void handle_create_image(int r); + + void get_parent_global_image_id(); + void handle_get_parent_global_image_id(int r); + + void get_local_parent_image_id(); + void handle_get_local_parent_image_id(int r); + + void open_remote_parent_image(); + void handle_open_remote_parent_image(int r); + + void clone_image(); + void handle_clone_image(int r); + + void close_remote_parent_image(); + void handle_close_remote_parent_image(int r); + + void error(int r); + void finish(int r); + + int validate_parent(); + + void populate_image_options(librbd::ImageOptions* image_options); + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::CreateImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_CREATE_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.cc b/src/tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.cc new file mode 100644 index 000000000..74e975373 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.cc @@ -0,0 +1,85 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_client.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::" \ + << "GetMirrorImageIdRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_rados_callback; + +template <typename I> +void GetMirrorImageIdRequest<I>::send() { + dout(20) << dendl; + get_image_id(); +} + +template <typename I> +void GetMirrorImageIdRequest<I>::get_image_id() { + dout(20) << dendl; + + // attempt to cross-reference a image id by the global image id + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_image_id_start(&op, m_global_image_id); + + librados::AioCompletion *aio_comp = create_rados_callback< + GetMirrorImageIdRequest<I>, + &GetMirrorImageIdRequest<I>::handle_get_image_id>( + this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void GetMirrorImageIdRequest<I>::handle_get_image_id(int r) { + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_get_image_id_finish( + &iter, m_image_id); + } + + dout(20) << "r=" << r << ", " + << "image_id=" << *m_image_id << dendl; + + if (r < 0) { + if (r == -ENOENT) { + dout(10) << "global image " << m_global_image_id << " not registered" + << dendl; + } else { + derr << "failed to retrieve image id: " << cpp_strerror(r) << dendl; + } + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void GetMirrorImageIdRequest<I>::finish(int r) { + dout(20) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::GetMirrorImageIdRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h b/src/tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h new file mode 100644 index 000000000..b26645138 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h @@ -0,0 +1,75 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_GET_MIRROR_IMAGE_ID_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_GET_MIRROR_IMAGE_ID_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados_fwd.hpp" +#include <string> + +namespace librbd { struct ImageCtx; } + +struct Context; + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <typename ImageCtxT = librbd::ImageCtx> +class GetMirrorImageIdRequest { +public: + static GetMirrorImageIdRequest *create(librados::IoCtx &io_ctx, + const std::string &global_image_id, + std::string *image_id, + Context *on_finish) { + return new GetMirrorImageIdRequest(io_ctx, global_image_id, image_id, + on_finish); + } + + GetMirrorImageIdRequest(librados::IoCtx &io_ctx, + const std::string &global_image_id, + std::string *image_id, + Context *on_finish) + : m_io_ctx(io_ctx), m_global_image_id(global_image_id), + m_image_id(image_id), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * GET_IMAGE_ID + * | + * v + * <finish> + + * @endverbatim + */ + + librados::IoCtx &m_io_ctx; + std::string m_global_image_id; + std::string *m_image_id; + Context *m_on_finish; + + bufferlist m_out_bl; + + void get_image_id(); + void handle_get_image_id(int r); + + void finish(int r); + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::GetMirrorImageIdRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_GET_MIRROR_IMAGE_ID_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/OpenImageRequest.cc b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.cc new file mode 100644 index 000000000..e6ab382be --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.cc @@ -0,0 +1,79 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "OpenImageRequest.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Utils.h" +#include <type_traits> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::OpenImageRequest: " \ + << this << " " << __func__ << " " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_context_callback; + +template <typename I> +OpenImageRequest<I>::OpenImageRequest(librados::IoCtx &io_ctx, I **image_ctx, + const std::string &image_id, + bool read_only, Context *on_finish) + : m_io_ctx(io_ctx), m_image_ctx(image_ctx), m_image_id(image_id), + m_read_only(read_only), m_on_finish(on_finish) { +} + +template <typename I> +void OpenImageRequest<I>::send() { + send_open_image(); +} + +template <typename I> +void OpenImageRequest<I>::send_open_image() { + dout(20) << dendl; + + *m_image_ctx = I::create("", m_image_id, nullptr, m_io_ctx, m_read_only); + + if (!m_read_only) { + // ensure non-primary images can be modified + (*m_image_ctx)->read_only_mask = ~librbd::IMAGE_READ_ONLY_FLAG_NON_PRIMARY; + } + + Context *ctx = create_context_callback< + OpenImageRequest<I>, &OpenImageRequest<I>::handle_open_image>( + this); + (*m_image_ctx)->state->open(0, ctx); +} + +template <typename I> +void OpenImageRequest<I>::handle_open_image(int r) { + dout(20) << ": r=" << r << dendl; + + if (r < 0) { + derr << ": failed to open image '" << m_image_id << "': " + << cpp_strerror(r) << dendl; + *m_image_ctx = nullptr; + } + + finish(r); +} + +template <typename I> +void OpenImageRequest<I>::finish(int r) { + dout(20) << ": r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::OpenImageRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/OpenImageRequest.h b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.h new file mode 100644 index 000000000..01ab31171 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/OpenImageRequest.h @@ -0,0 +1,71 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_OPEN_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_OPEN_IMAGE_REQUEST_H + +#include "include/int_types.h" +#include "librbd/ImageCtx.h" +#include <string> + +class Context; +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <typename ImageCtxT = librbd::ImageCtx> +class OpenImageRequest { +public: + static OpenImageRequest* create(librados::IoCtx &io_ctx, + ImageCtxT **image_ctx, + const std::string &image_id, + bool read_only, Context *on_finish) { + return new OpenImageRequest(io_ctx, image_ctx, image_id, read_only, + on_finish); + } + + OpenImageRequest(librados::IoCtx &io_ctx, ImageCtxT **image_ctx, + const std::string &image_id, bool read_only, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * OPEN_IMAGE + * | + * v + * <finish> + * + * @endverbatim + */ + librados::IoCtx &m_io_ctx; + ImageCtxT **m_image_ctx; + std::string m_image_id; + bool m_read_only; + Context *m_on_finish; + + void send_open_image(); + void handle_open_image(int r); + + void send_close_image(int r); + void handle_close_image(int r); + + void finish(int r); + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::OpenImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_OPEN_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc new file mode 100644 index 000000000..7f8d9608e --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc @@ -0,0 +1,292 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/compat.h" +#include "CloseImageRequest.h" +#include "OpenLocalImageRequest.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/exclusive_lock/Policy.h" +#include "librbd/journal/Policy.h" +#include "librbd/mirror/GetInfoRequest.h" +#include <type_traits> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::OpenLocalImageRequest: " \ + << this << " " << __func__ << " " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_context_callback; + +namespace { + +template <typename I> +struct MirrorExclusiveLockPolicy : public librbd::exclusive_lock::Policy { + I *image_ctx; + + MirrorExclusiveLockPolicy(I *image_ctx) : image_ctx(image_ctx) { + } + + bool may_auto_request_lock() override { + return false; + } + + int lock_requested(bool force) override { + int r = -EROFS; + { + std::shared_lock owner_locker{image_ctx->owner_lock}; + std::shared_lock image_locker{image_ctx->image_lock}; + if (image_ctx->journal == nullptr || image_ctx->journal->is_tag_owner()) { + r = 0; + } + } + + if (r == 0) { + // if the local image journal has been closed or if it was (force) + // promoted allow the lock to be released to another client + image_ctx->exclusive_lock->release_lock(nullptr); + } + return r; + } + + bool accept_blocked_request( + librbd::exclusive_lock::OperationRequestType request_type) override { + switch (request_type) { + case librbd::exclusive_lock::OPERATION_REQUEST_TYPE_TRASH_SNAP_REMOVE: + case librbd::exclusive_lock::OPERATION_REQUEST_TYPE_FORCE_PROMOTION: + return true; + default: + return false; + } + } +}; + +struct MirrorJournalPolicy : public librbd::journal::Policy { + librbd::asio::ContextWQ *work_queue; + + MirrorJournalPolicy(librbd::asio::ContextWQ *work_queue) + : work_queue(work_queue) { + } + + bool append_disabled() const override { + // avoid recording any events to the local journal + return true; + } + bool journal_disabled() const override { + return false; + } + + void allocate_tag_on_lock(Context *on_finish) override { + // rbd-mirror will manually create tags by copying them from the peer + work_queue->queue(on_finish, 0); + } +}; + +} // anonymous namespace + +template <typename I> +OpenLocalImageRequest<I>::OpenLocalImageRequest( + librados::IoCtx &local_io_ctx, + I **local_image_ctx, + const std::string &local_image_id, + librbd::asio::ContextWQ *work_queue, + Context *on_finish) + : m_local_io_ctx(local_io_ctx), m_local_image_ctx(local_image_ctx), + m_local_image_id(local_image_id), m_work_queue(work_queue), + m_on_finish(on_finish) { +} + +template <typename I> +void OpenLocalImageRequest<I>::send() { + send_open_image(); +} + +template <typename I> +void OpenLocalImageRequest<I>::send_open_image() { + dout(20) << dendl; + + *m_local_image_ctx = I::create("", m_local_image_id, nullptr, + m_local_io_ctx, false); + + // ensure non-primary images can be modified + (*m_local_image_ctx)->read_only_mask = + ~librbd::IMAGE_READ_ONLY_FLAG_NON_PRIMARY; + + { + std::scoped_lock locker{(*m_local_image_ctx)->owner_lock, + (*m_local_image_ctx)->image_lock}; + (*m_local_image_ctx)->set_exclusive_lock_policy( + new MirrorExclusiveLockPolicy<I>(*m_local_image_ctx)); + (*m_local_image_ctx)->set_journal_policy( + new MirrorJournalPolicy(m_work_queue)); + } + + Context *ctx = create_context_callback< + OpenLocalImageRequest<I>, &OpenLocalImageRequest<I>::handle_open_image>( + this); + (*m_local_image_ctx)->state->open(0, ctx); +} + +template <typename I> +void OpenLocalImageRequest<I>::handle_open_image(int r) { + dout(20) << ": r=" << r << dendl; + + if (r < 0) { + if (r == -ENOENT) { + dout(10) << ": local image does not exist" << dendl; + } else { + derr << ": failed to open image '" << m_local_image_id << "': " + << cpp_strerror(r) << dendl; + } + *m_local_image_ctx = nullptr; + finish(r); + return; + } + + send_get_mirror_info(); +} + +template <typename I> +void OpenLocalImageRequest<I>::send_get_mirror_info() { + dout(20) << dendl; + + Context *ctx = create_context_callback< + OpenLocalImageRequest<I>, + &OpenLocalImageRequest<I>::handle_get_mirror_info>( + this); + auto request = librbd::mirror::GetInfoRequest<I>::create( + **m_local_image_ctx, &m_mirror_image, &m_promotion_state, + &m_primary_mirror_uuid, ctx); + request->send(); +} + +template <typename I> +void OpenLocalImageRequest<I>::handle_get_mirror_info(int r) { + dout(20) << ": r=" << r << dendl; + + if (r == -ENOENT) { + dout(5) << ": local image is not mirrored" << dendl; + send_close_image(r); + return; + } else if (r < 0) { + derr << ": error querying local image primary status: " << cpp_strerror(r) + << dendl; + send_close_image(r); + return; + } + + if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_DISABLING) { + dout(5) << ": local image mirroring is being disabled" << dendl; + send_close_image(-ENOENT); + return; + } + + // if the local image owns the tag -- don't steal the lock since + // we aren't going to mirror peer data into this image anyway + if (m_promotion_state == librbd::mirror::PROMOTION_STATE_PRIMARY) { + dout(10) << ": local image is primary -- skipping image replay" << dendl; + send_close_image(-EREMOTEIO); + return; + } + + send_lock_image(); +} + +template <typename I> +void OpenLocalImageRequest<I>::send_lock_image() { + std::shared_lock owner_locker{(*m_local_image_ctx)->owner_lock}; + if ((*m_local_image_ctx)->exclusive_lock == nullptr) { + owner_locker.unlock(); + if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) { + finish(0); + } else { + derr << ": image does not support exclusive lock" << dendl; + send_close_image(-EINVAL); + } + return; + } + + dout(20) << dendl; + + // disallow any proxied maintenance operations before grabbing lock + (*m_local_image_ctx)->exclusive_lock->block_requests(-EROFS); + + Context *ctx = create_context_callback< + OpenLocalImageRequest<I>, &OpenLocalImageRequest<I>::handle_lock_image>( + this); + + (*m_local_image_ctx)->exclusive_lock->acquire_lock(ctx); +} + +template <typename I> +void OpenLocalImageRequest<I>::handle_lock_image(int r) { + dout(20) << ": r=" << r << dendl; + + if (r < 0) { + derr << ": failed to lock image '" << m_local_image_id << "': " + << cpp_strerror(r) << dendl; + send_close_image(r); + return; + } + + { + std::shared_lock owner_locker{(*m_local_image_ctx)->owner_lock}; + if ((*m_local_image_ctx)->exclusive_lock == nullptr || + !(*m_local_image_ctx)->exclusive_lock->is_lock_owner()) { + derr << ": image is not locked" << dendl; + send_close_image(-EBUSY); + return; + } + } + + finish(0); +} + +template <typename I> +void OpenLocalImageRequest<I>::send_close_image(int r) { + dout(20) << dendl; + + if (m_ret_val == 0 && r < 0) { + m_ret_val = r; + } + + Context *ctx = create_context_callback< + OpenLocalImageRequest<I>, &OpenLocalImageRequest<I>::handle_close_image>( + this); + CloseImageRequest<I> *request = CloseImageRequest<I>::create( + m_local_image_ctx, ctx); + request->send(); +} + +template <typename I> +void OpenLocalImageRequest<I>::handle_close_image(int r) { + dout(20) << dendl; + + ceph_assert(r == 0); + finish(m_ret_val); +} + +template <typename I> +void OpenLocalImageRequest<I>::finish(int r) { + dout(20) << ": r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::OpenLocalImageRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h new file mode 100644 index 000000000..9a642bc39 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.h @@ -0,0 +1,97 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_OPEN_LOCAL_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_OPEN_LOCAL_IMAGE_REQUEST_H + +#include "include/int_types.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/mirror/Types.h" +#include <string> + +class Context; +namespace librbd { +class ImageCtx; +namespace asio { struct ContextWQ; } +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <typename ImageCtxT = librbd::ImageCtx> +class OpenLocalImageRequest { +public: + static OpenLocalImageRequest* create(librados::IoCtx &local_io_ctx, + ImageCtxT **local_image_ctx, + const std::string &local_image_id, + librbd::asio::ContextWQ *work_queue, + Context *on_finish) { + return new OpenLocalImageRequest(local_io_ctx, local_image_ctx, + local_image_id, work_queue, on_finish); + } + + OpenLocalImageRequest(librados::IoCtx &local_io_ctx, + ImageCtxT **local_image_ctx, + const std::string &local_image_id, + librbd::asio::ContextWQ *work_queue, + Context *on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * OPEN_IMAGE * * * * * * * * + * | * + * v * + * GET_MIRROR_INFO * * * * * + * | * + * v (skip if primary) v + * LOCK_IMAGE * * * > CLOSE_IMAGE + * | | + * v | + * <finish> <---------------/ + * + * @endverbatim + */ + librados::IoCtx &m_local_io_ctx; + ImageCtxT **m_local_image_ctx; + std::string m_local_image_id; + librbd::asio::ContextWQ *m_work_queue; + Context *m_on_finish; + + cls::rbd::MirrorImage m_mirror_image; + librbd::mirror::PromotionState m_promotion_state = + librbd::mirror::PROMOTION_STATE_NON_PRIMARY; + std::string m_primary_mirror_uuid; + int m_ret_val = 0; + + void send_open_image(); + void handle_open_image(int r); + + void send_get_mirror_info(); + void handle_get_mirror_info(int r); + + void send_lock_image(); + void handle_lock_image(int r); + + void send_close_image(int r); + void handle_close_image(int r); + + void finish(int r); + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::OpenLocalImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_OPEN_LOCAL_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc new file mode 100644 index 000000000..b1fef7254 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc @@ -0,0 +1,197 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_client.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "tools/rbd_mirror/ImageDeleter.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" +#include <type_traits> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::" \ + << "PrepareLocalImageRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +void PrepareLocalImageRequest<I>::send() { + dout(10) << dendl; + get_local_image_id(); +} + +template <typename I> +void PrepareLocalImageRequest<I>::get_local_image_id() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + PrepareLocalImageRequest<I>, + &PrepareLocalImageRequest<I>::handle_get_local_image_id>(this); + auto req = GetMirrorImageIdRequest<I>::create(m_io_ctx, m_global_image_id, + &m_local_image_id, ctx); + req->send(); +} + +template <typename I> +void PrepareLocalImageRequest<I>::handle_get_local_image_id(int r) { + dout(10) << "r=" << r << ", " + << "local_image_id=" << m_local_image_id << dendl; + + if (r < 0) { + finish(r); + return; + } + + get_local_image_name(); +} + +template <typename I> +void PrepareLocalImageRequest<I>::get_local_image_name() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::dir_get_name_start(&op, m_local_image_id); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + PrepareLocalImageRequest<I>, + &PrepareLocalImageRequest<I>::handle_get_local_image_name>(this); + int r = m_io_ctx.aio_operate(RBD_DIRECTORY, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void PrepareLocalImageRequest<I>::handle_get_local_image_name(int r) { + dout(10) << "r=" << r << dendl; + + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::dir_get_name_finish(&it, m_local_image_name); + } + + if (r == -ENOENT) { + // proceed we should have a mirror image record if we got this far + dout(10) << "image does not exist for local image id " << m_local_image_id + << dendl; + *m_local_image_name = ""; + } else if (r < 0) { + derr << "failed to retrieve image name: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_mirror_info(); +} + +template <typename I> +void PrepareLocalImageRequest<I>::get_mirror_info() { + dout(10) << dendl; + + auto ctx = create_context_callback< + PrepareLocalImageRequest<I>, + &PrepareLocalImageRequest<I>::handle_get_mirror_info>(this); + auto req = librbd::mirror::GetInfoRequest<I>::create( + m_io_ctx, m_work_queue, m_local_image_id, &m_mirror_image, + &m_promotion_state, &m_primary_mirror_uuid, ctx); + req->send(); +} + +template <typename I> +void PrepareLocalImageRequest<I>::handle_get_mirror_info(int r) { + dout(10) << ": r=" << r << dendl; + + if (r < 0) { + derr << "failed to retrieve local mirror image info: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_CREATING) { + dout(5) << "local image is still in creating state, issuing a removal" + << dendl; + move_to_trash(); + return; + } else if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_DISABLING) { + dout(5) << "local image mirroring is in disabling state" << dendl; + finish(-ERESTART); + return; + } + + switch (m_mirror_image.mode) { + case cls::rbd::MIRROR_IMAGE_MODE_JOURNAL: + // journal-based local image exists + { + auto state_builder = journal::StateBuilder<I>::create(m_global_image_id); + state_builder->local_primary_mirror_uuid = m_primary_mirror_uuid; + *m_state_builder = state_builder; + } + break; + case cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT: + // snapshot-based local image exists + *m_state_builder = snapshot::StateBuilder<I>::create(m_global_image_id); + break; + default: + derr << "unsupported mirror image mode " << m_mirror_image.mode << " " + << "for image " << m_global_image_id << dendl; + finish(-EOPNOTSUPP); + break; + } + + dout(10) << "local_image_id=" << m_local_image_id << ", " + << "local_promotion_state=" << m_promotion_state << ", " + << "local_primary_mirror_uuid=" << m_primary_mirror_uuid << dendl; + (*m_state_builder)->local_image_id = m_local_image_id; + (*m_state_builder)->local_promotion_state = m_promotion_state; + finish(0); +} + +template <typename I> +void PrepareLocalImageRequest<I>::move_to_trash() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + PrepareLocalImageRequest<I>, + &PrepareLocalImageRequest<I>::handle_move_to_trash>(this); + ImageDeleter<I>::trash_move(m_io_ctx, m_global_image_id, + false, m_work_queue, ctx); +} + +template <typename I> +void PrepareLocalImageRequest<I>::handle_move_to_trash(int r) { + dout(10) << ": r=" << r << dendl; + + finish(-ENOENT); +} + +template <typename I> +void PrepareLocalImageRequest<I>::finish(int r) { + dout(10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::PrepareLocalImageRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h new file mode 100644 index 000000000..6372169ff --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h @@ -0,0 +1,115 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_PREPARE_LOCAL_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_PREPARE_LOCAL_IMAGE_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados_fwd.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" +#include <string> + +struct Context; + +namespace librbd { +struct ImageCtx; +namespace asio { struct ContextWQ; } +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <typename> class StateBuilder; + +template <typename ImageCtxT = librbd::ImageCtx> +class PrepareLocalImageRequest { +public: + static PrepareLocalImageRequest *create( + librados::IoCtx &io_ctx, + const std::string &global_image_id, + std::string *local_image_name, + StateBuilder<ImageCtxT>** state_builder, + librbd::asio::ContextWQ *work_queue, + Context *on_finish) { + return new PrepareLocalImageRequest(io_ctx, global_image_id, + local_image_name, state_builder, + work_queue, on_finish); + } + + PrepareLocalImageRequest( + librados::IoCtx &io_ctx, + const std::string &global_image_id, + std::string *local_image_name, + StateBuilder<ImageCtxT>** state_builder, + librbd::asio::ContextWQ *work_queue, + Context *on_finish) + : m_io_ctx(io_ctx), m_global_image_id(global_image_id), + m_local_image_name(local_image_name), m_state_builder(state_builder), + m_work_queue(work_queue), m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * GET_LOCAL_IMAGE_ID + * | + * v + * GET_LOCAL_IMAGE_NAME + * | + * v + * GET_MIRROR_INFO + * | + * | (if the image mirror state is CREATING) + * v + * TRASH_MOVE + * | + * v + * <finish> + * + * @endverbatim + */ + + librados::IoCtx &m_io_ctx; + std::string m_global_image_id; + std::string *m_local_image_name; + StateBuilder<ImageCtxT>** m_state_builder; + librbd::asio::ContextWQ *m_work_queue; + Context *m_on_finish; + + bufferlist m_out_bl; + std::string m_local_image_id; + cls::rbd::MirrorImage m_mirror_image; + librbd::mirror::PromotionState m_promotion_state; + std::string m_primary_mirror_uuid; + + void get_local_image_id(); + void handle_get_local_image_id(int r); + + void get_local_image_name(); + void handle_get_local_image_name(int r); + + void get_mirror_info(); + void handle_get_mirror_info(int r); + + void move_to_trash(); + void handle_move_to_trash(int r); + + void finish(int r); + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::PrepareLocalImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_PREPARE_LOCAL_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc new file mode 100644 index 000000000..45a44a300 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc @@ -0,0 +1,283 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_client.h" +#include "common/debug.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "journal/Settings.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/mirror/GetInfoRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::" \ + << "PrepareRemoteImageRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +void PrepareRemoteImageRequest<I>::send() { + if (*m_state_builder != nullptr) { + (*m_state_builder)->remote_mirror_uuid = m_remote_pool_meta.mirror_uuid; + auto state_builder = dynamic_cast<snapshot::StateBuilder<I>*>(*m_state_builder); + if (state_builder) { + state_builder->remote_mirror_peer_uuid = m_remote_pool_meta.mirror_peer_uuid; + } + } + + get_remote_image_id(); +} + +template <typename I> +void PrepareRemoteImageRequest<I>::get_remote_image_id() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + PrepareRemoteImageRequest<I>, + &PrepareRemoteImageRequest<I>::handle_get_remote_image_id>(this); + auto req = GetMirrorImageIdRequest<I>::create(m_remote_io_ctx, + m_global_image_id, + &m_remote_image_id, ctx); + req->send(); +} + +template <typename I> +void PrepareRemoteImageRequest<I>::handle_get_remote_image_id(int r) { + dout(10) << "r=" << r << ", " + << "remote_image_id=" << m_remote_image_id << dendl; + + if (r < 0) { + finish(r); + return; + } + + get_mirror_info(); +} + +template <typename I> +void PrepareRemoteImageRequest<I>::get_mirror_info() { + dout(10) << dendl; + + auto ctx = create_context_callback< + PrepareRemoteImageRequest<I>, + &PrepareRemoteImageRequest<I>::handle_get_mirror_info>(this); + auto req = librbd::mirror::GetInfoRequest<I>::create( + m_remote_io_ctx, m_threads->work_queue, m_remote_image_id, + &m_mirror_image, &m_promotion_state, &m_primary_mirror_uuid, + ctx); + req->send(); +} + +template <typename I> +void PrepareRemoteImageRequest<I>::handle_get_mirror_info(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -ENOENT) { + dout(10) << "image " << m_global_image_id << " not mirrored" << dendl; + finish(r); + return; + } else if (r < 0) { + derr << "failed to retrieve mirror image details for image " + << m_global_image_id << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + auto state_builder = *m_state_builder; + if (state_builder != nullptr && + state_builder->get_mirror_image_mode() != m_mirror_image.mode) { + derr << "local and remote mirror image using different mirroring modes " + << "for image " << m_global_image_id << ": split-brain" << dendl; + finish(-EEXIST); + return; + } else if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_DISABLING) { + dout(5) << "remote image mirroring is being disabled" << dendl; + finish(-ENOENT); + return; + } + + switch (m_mirror_image.mode) { + case cls::rbd::MIRROR_IMAGE_MODE_JOURNAL: + get_client(); + break; + case cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT: + finalize_snapshot_state_builder(); + finish(0); + break; + default: + derr << "unsupported mirror image mode " << m_mirror_image.mode << " " + << "for image " << m_global_image_id << dendl; + finish(-EOPNOTSUPP); + break; + } +} + +template <typename I> +void PrepareRemoteImageRequest<I>::get_client() { + dout(10) << dendl; + + auto cct = static_cast<CephContext *>(m_local_io_ctx.cct()); + ::journal::Settings journal_settings; + journal_settings.commit_interval = cct->_conf.get_val<double>( + "rbd_mirror_journal_commit_age"); + + // TODO use Journal thread pool for journal ops until converted to ASIO + ContextWQ* context_wq; + librbd::Journal<>::get_work_queue(cct, &context_wq); + + ceph_assert(m_remote_journaler == nullptr); + m_remote_journaler = new Journaler(context_wq, m_threads->timer, + &m_threads->timer_lock, m_remote_io_ctx, + m_remote_image_id, m_local_mirror_uuid, + journal_settings, m_cache_manager_handler); + + Context *ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + PrepareRemoteImageRequest<I>, + &PrepareRemoteImageRequest<I>::handle_get_client>(this)); + m_remote_journaler->get_client(m_local_mirror_uuid, &m_client, ctx); +} + +template <typename I> +void PrepareRemoteImageRequest<I>::handle_get_client(int r) { + dout(10) << "r=" << r << dendl; + + MirrorPeerClientMeta client_meta; + if (r == -ENOENT) { + dout(10) << "client not registered" << dendl; + register_client(); + } else if (r < 0) { + derr << "failed to retrieve client: " << cpp_strerror(r) << dendl; + finish(r); + } else if (!util::decode_client_meta(m_client, &client_meta)) { + // require operator intervention since the data is corrupt + finish(-EBADMSG); + } else { + // skip registration if it already exists + finalize_journal_state_builder(m_client.state, client_meta); + finish(0); + } +} + +template <typename I> +void PrepareRemoteImageRequest<I>::register_client() { + dout(10) << dendl; + + auto state_builder = *m_state_builder; + librbd::journal::MirrorPeerClientMeta client_meta{ + (state_builder == nullptr ? "" : state_builder->local_image_id)}; + client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + librbd::journal::ClientData client_data{client_meta}; + bufferlist client_data_bl; + encode(client_data, client_data_bl); + + Context *ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + PrepareRemoteImageRequest<I>, + &PrepareRemoteImageRequest<I>::handle_register_client>(this)); + m_remote_journaler->register_client(client_data_bl, ctx); +} + +template <typename I> +void PrepareRemoteImageRequest<I>::handle_register_client(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to register with remote journal: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + auto state_builder = *m_state_builder; + librbd::journal::MirrorPeerClientMeta client_meta{ + (state_builder == nullptr ? "" : state_builder->local_image_id)}; + client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + finalize_journal_state_builder(cls::journal::CLIENT_STATE_CONNECTED, + client_meta); + finish(0); +} + +template <typename I> +void PrepareRemoteImageRequest<I>::finalize_journal_state_builder( + cls::journal::ClientState client_state, + const MirrorPeerClientMeta& client_meta) { + journal::StateBuilder<I>* state_builder = nullptr; + if (*m_state_builder != nullptr) { + // already verified that it's a matching builder in + // 'handle_get_mirror_info' + state_builder = dynamic_cast<journal::StateBuilder<I>*>(*m_state_builder); + ceph_assert(state_builder != nullptr); + } else { + state_builder = journal::StateBuilder<I>::create(m_global_image_id); + *m_state_builder = state_builder; + } + + state_builder->remote_mirror_uuid = m_remote_pool_meta.mirror_uuid; + state_builder->remote_image_id = m_remote_image_id; + state_builder->remote_promotion_state = m_promotion_state; + state_builder->remote_journaler = m_remote_journaler; + state_builder->remote_client_state = client_state; + state_builder->remote_client_meta = client_meta; +} + +template <typename I> +void PrepareRemoteImageRequest<I>::finalize_snapshot_state_builder() { + snapshot::StateBuilder<I>* state_builder = nullptr; + if (*m_state_builder != nullptr) { + state_builder = dynamic_cast<snapshot::StateBuilder<I>*>(*m_state_builder); + ceph_assert(state_builder != nullptr); + } else { + state_builder = snapshot::StateBuilder<I>::create(m_global_image_id); + *m_state_builder = state_builder; + } + + dout(10) << "remote_mirror_uuid=" << m_remote_pool_meta.mirror_uuid << ", " + << "remote_mirror_peer_uuid=" + << m_remote_pool_meta.mirror_peer_uuid << ", " + << "remote_image_id=" << m_remote_image_id << ", " + << "remote_promotion_state=" << m_promotion_state << dendl; + state_builder->remote_mirror_uuid = m_remote_pool_meta.mirror_uuid; + state_builder->remote_mirror_peer_uuid = m_remote_pool_meta.mirror_peer_uuid; + state_builder->remote_image_id = m_remote_image_id; + state_builder->remote_promotion_state = m_promotion_state; +} + +template <typename I> +void PrepareRemoteImageRequest<I>::finish(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + delete m_remote_journaler; + m_remote_journaler = nullptr; + } + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::PrepareRemoteImageRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h new file mode 100644 index 000000000..483cfc001 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.h @@ -0,0 +1,153 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_PREPARE_REMOTE_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_PREPARE_REMOTE_IMAGE_REQUEST_H + +#include "include/buffer_fwd.h" +#include "include/rados/librados_fwd.hpp" +#include "cls/journal/cls_journal_types.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "librbd/mirror/Types.h" +#include "tools/rbd_mirror/Types.h" +#include <string> + +namespace journal { class Journaler; } +namespace journal { struct CacheManagerHandler; } +namespace librbd { struct ImageCtx; } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } + +struct Context; + +namespace rbd { +namespace mirror { + +template <typename> struct Threads; + +namespace image_replayer { + +template <typename> class StateBuilder; + +template <typename ImageCtxT = librbd::ImageCtx> +class PrepareRemoteImageRequest { +public: + typedef librbd::journal::TypeTraits<ImageCtxT> TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + typedef librbd::journal::MirrorPeerClientMeta MirrorPeerClientMeta; + + static PrepareRemoteImageRequest *create( + Threads<ImageCtxT> *threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const RemotePoolMeta& remote_pool_meta, + ::journal::CacheManagerHandler *cache_manager_handler, + StateBuilder<ImageCtxT>** state_builder, + Context *on_finish) { + return new PrepareRemoteImageRequest(threads, local_io_ctx, remote_io_ctx, + global_image_id, local_mirror_uuid, + remote_pool_meta, + cache_manager_handler, state_builder, + on_finish); + } + + PrepareRemoteImageRequest( + Threads<ImageCtxT> *threads, + librados::IoCtx &local_io_ctx, + librados::IoCtx &remote_io_ctx, + const std::string &global_image_id, + const std::string &local_mirror_uuid, + const RemotePoolMeta& remote_pool_meta, + ::journal::CacheManagerHandler *cache_manager_handler, + StateBuilder<ImageCtxT>** state_builder, + Context *on_finish) + : m_threads(threads), + m_local_io_ctx(local_io_ctx), + m_remote_io_ctx(remote_io_ctx), + m_global_image_id(global_image_id), + m_local_mirror_uuid(local_mirror_uuid), + m_remote_pool_meta(remote_pool_meta), + m_cache_manager_handler(cache_manager_handler), + m_state_builder(state_builder), + m_on_finish(on_finish) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * GET_REMOTE_IMAGE_ID + * | + * v + * GET_REMOTE_MIRROR_INFO + * | + * | (journal) + * \-----------> GET_CLIENT + * | | + * | v (skip if not needed) + * | REGISTER_CLIENT + * | | + * | | + * |/----------------/ + * | + * v + * <finish> + * + * @endverbatim + */ + + Threads<ImageCtxT> *m_threads; + librados::IoCtx &m_local_io_ctx; + librados::IoCtx &m_remote_io_ctx; + std::string m_global_image_id; + std::string m_local_mirror_uuid; + RemotePoolMeta m_remote_pool_meta; + ::journal::CacheManagerHandler *m_cache_manager_handler; + StateBuilder<ImageCtxT>** m_state_builder; + Context *m_on_finish; + + bufferlist m_out_bl; + std::string m_remote_image_id; + cls::rbd::MirrorImage m_mirror_image; + librbd::mirror::PromotionState m_promotion_state = + librbd::mirror::PROMOTION_STATE_UNKNOWN; + std::string m_primary_mirror_uuid; + + // journal-based mirroring + Journaler *m_remote_journaler = nullptr; + cls::journal::Client m_client; + + void get_remote_image_id(); + void handle_get_remote_image_id(int r); + + void get_mirror_info(); + void handle_get_mirror_info(int r); + + void get_client(); + void handle_get_client(int r); + + void register_client(); + void handle_register_client(int r); + + void finalize_journal_state_builder(cls::journal::ClientState client_state, + const MirrorPeerClientMeta& client_meta); + void finalize_snapshot_state_builder(); + + void finish(int r); +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::PrepareRemoteImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_PREPARE_REMOTE_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/Replayer.h b/src/tools/rbd_mirror/image_replayer/Replayer.h new file mode 100644 index 000000000..f3bfa4da0 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/Replayer.h @@ -0,0 +1,39 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_H +#define RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_H + +#include <string> + +struct Context; + +namespace rbd { +namespace mirror { +namespace image_replayer { + +struct Replayer { + virtual ~Replayer() {} + + virtual void destroy() = 0; + + virtual void init(Context* on_finish) = 0; + virtual void shut_down(Context* on_finish) = 0; + + virtual void flush(Context* on_finish) = 0; + + virtual bool get_replay_status(std::string* description, + Context* on_finish) = 0; + + virtual bool is_replaying() const = 0; + virtual bool is_resync_requested() const = 0; + + virtual int get_error_code() const = 0; + virtual std::string get_error_description() const = 0; +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_H diff --git a/src/tools/rbd_mirror/image_replayer/ReplayerListener.h b/src/tools/rbd_mirror/image_replayer/ReplayerListener.h new file mode 100644 index 000000000..f17f401b1 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/ReplayerListener.h @@ -0,0 +1,21 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_LISTENER_H +#define RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_LISTENER_H + +namespace rbd { +namespace mirror { +namespace image_replayer { + +struct ReplayerListener { + virtual ~ReplayerListener() {} + + virtual void handle_notification() = 0; +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_REPLAYER_REPLAYER_LISTENER_H diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/StateBuilder.cc new file mode 100644 index 000000000..55fb3509d --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.cc @@ -0,0 +1,138 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "StateBuilder.h" +#include "include/ceph_assert.h" +#include "include/Context.h" +#include "common/debug.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_sync/Types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::" \ + << "StateBuilder: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { + +template <typename I> +StateBuilder<I>::StateBuilder(const std::string& global_image_id) + : global_image_id(global_image_id) { + dout(10) << "global_image_id=" << global_image_id << dendl; +} + +template <typename I> +StateBuilder<I>::~StateBuilder() { + ceph_assert(local_image_ctx == nullptr); + ceph_assert(remote_image_ctx == nullptr); + ceph_assert(m_sync_point_handler == nullptr); +} + +template <typename I> +bool StateBuilder<I>::is_local_primary() const { + if (local_promotion_state == librbd::mirror::PROMOTION_STATE_PRIMARY) { + ceph_assert(!local_image_id.empty()); + return true; + } + return false; +} + +template <typename I> +bool StateBuilder<I>::is_remote_primary() const { + if (remote_promotion_state == librbd::mirror::PROMOTION_STATE_PRIMARY) { + ceph_assert(!remote_image_id.empty()); + return true; + } + return false; +} + +template <typename I> +bool StateBuilder<I>::is_linked() const { + if (local_promotion_state == librbd::mirror::PROMOTION_STATE_NON_PRIMARY) { + ceph_assert(!local_image_id.empty()); + return is_linked_impl(); + } + return false; +} + +template <typename I> +void StateBuilder<I>::close_local_image(Context* on_finish) { + if (local_image_ctx == nullptr) { + on_finish->complete(0); + return; + } + + dout(10) << dendl; + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_close_local_image(r, on_finish); + }); + auto request = image_replayer::CloseImageRequest<I>::create( + &local_image_ctx, ctx); + request->send(); +} + +template <typename I> +void StateBuilder<I>::handle_close_local_image(int r, Context* on_finish) { + dout(10) << "r=" << r << dendl; + + ceph_assert(local_image_ctx == nullptr); + if (r < 0) { + derr << "failed to close local image for image " << global_image_id << ": " + << cpp_strerror(r) << dendl; + } + + on_finish->complete(r); +} + +template <typename I> +void StateBuilder<I>::close_remote_image(Context* on_finish) { + if (remote_image_ctx == nullptr) { + on_finish->complete(0); + return; + } + + dout(10) << dendl; + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_close_remote_image(r, on_finish); + }); + auto request = image_replayer::CloseImageRequest<I>::create( + &remote_image_ctx, ctx); + request->send(); +} + +template <typename I> +void StateBuilder<I>::handle_close_remote_image(int r, Context* on_finish) { + dout(10) << "r=" << r << dendl; + + ceph_assert(remote_image_ctx == nullptr); + if (r < 0) { + derr << "failed to close remote image for image " << global_image_id << ": " + << cpp_strerror(r) << dendl; + } + + on_finish->complete(r); +} + +template <typename I> +void StateBuilder<I>::destroy_sync_point_handler() { + if (m_sync_point_handler == nullptr) { + return; + } + + dout(15) << dendl; + m_sync_point_handler->destroy(); + m_sync_point_handler = nullptr; +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::StateBuilder<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h new file mode 100644 index 000000000..51cf8668c --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h @@ -0,0 +1,114 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_REPLAYER_STATE_BUILDER_H +#define CEPH_RBD_MIRROR_IMAGE_REPLAYER_STATE_BUILDER_H + +#include "include/rados/librados_fwd.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/Types.h" + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { + +struct BaseRequest; +template <typename> class InstanceWatcher; +struct PoolMetaCache; +struct ProgressContext; +template <typename> class Threads; + +namespace image_sync { struct SyncPointHandler; } + +namespace image_replayer { + +struct Replayer; +struct ReplayerListener; + +template <typename ImageCtxT> +class StateBuilder { +public: + StateBuilder(const StateBuilder&) = delete; + StateBuilder& operator=(const StateBuilder&) = delete; + + virtual ~StateBuilder(); + + virtual void destroy() { + delete this; + } + + virtual void close(Context* on_finish) = 0; + + virtual bool is_disconnected() const = 0; + + bool is_local_primary() const; + bool is_remote_primary() const; + bool is_linked() const; + + virtual cls::rbd::MirrorImageMode get_mirror_image_mode() const = 0; + + virtual image_sync::SyncPointHandler* create_sync_point_handler() = 0; + void destroy_sync_point_handler(); + + virtual bool replay_requires_remote_image() const = 0; + + void close_remote_image(Context* on_finish); + + virtual BaseRequest* create_local_image_request( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + Context* on_finish) = 0; + + virtual BaseRequest* create_prepare_replay_request( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + bool* resync_requested, + bool* syncing, + Context* on_finish) = 0; + + virtual Replayer* create_replayer( + Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + ReplayerListener* replayer_listener) = 0; + + std::string global_image_id; + + std::string local_image_id; + librbd::mirror::PromotionState local_promotion_state = + librbd::mirror::PROMOTION_STATE_UNKNOWN; + ImageCtxT* local_image_ctx = nullptr; + + std::string remote_mirror_uuid; + std::string remote_image_id; + librbd::mirror::PromotionState remote_promotion_state = + librbd::mirror::PROMOTION_STATE_UNKNOWN; + ImageCtxT* remote_image_ctx = nullptr; + +protected: + image_sync::SyncPointHandler* m_sync_point_handler = nullptr; + + StateBuilder(const std::string& global_image_id); + + void close_local_image(Context* on_finish); + +private: + virtual bool is_linked_impl() const = 0; + + void handle_close_local_image(int r, Context* on_finish); + void handle_close_remote_image(int r, Context* on_finish); +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::StateBuilder<librbd::ImageCtx>; + +#endif // CEPH_RBD_MIRROR_IMAGE_REPLAYER_STATE_BUILDER_H diff --git a/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc new file mode 100644 index 000000000..5d9c9aca1 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.cc @@ -0,0 +1,34 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h" +#include "common/Clock.h" + +namespace rbd { +namespace mirror { +namespace image_replayer { + +void TimeRollingMean::operator()(uint32_t value) { + auto time = ceph_clock_now(); + if (m_last_time.is_zero()) { + m_last_time = time; + } else if (m_last_time.sec() < time.sec()) { + auto sec = m_last_time.sec(); + while (sec++ < time.sec()) { + m_rolling_mean(m_sum); + m_sum = 0; + } + + m_last_time = time; + } + + m_sum += value; +} + +double TimeRollingMean::get_average() const { + return boost::accumulators::rolling_mean(m_rolling_mean); +} + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h new file mode 100644 index 000000000..139ef893f --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/TimeRollingMean.h @@ -0,0 +1,40 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H +#define RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H + +#include "include/utime.h" +#include <boost/accumulators/accumulators.hpp> +#include <boost/accumulators/statistics/stats.hpp> +#include <boost/accumulators/statistics/rolling_mean.hpp> + +namespace rbd { +namespace mirror { +namespace image_replayer { + +class TimeRollingMean { +public: + + void operator()(uint32_t value); + + double get_average() const; + +private: + typedef boost::accumulators::accumulator_set< + uint64_t, boost::accumulators::stats< + boost::accumulators::tag::rolling_mean>> RollingMean; + + utime_t m_last_time; + uint64_t m_sum = 0; + + RollingMean m_rolling_mean{ + boost::accumulators::tag::rolling_window::window_size = 30}; + +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_REPLAYER_TIME_ROLLING_MEAN_H diff --git a/src/tools/rbd_mirror/image_replayer/Types.h b/src/tools/rbd_mirror/image_replayer/Types.h new file mode 100644 index 000000000..6ab988a76 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/Types.h @@ -0,0 +1,21 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_REPLAYER_TYPES_H +#define CEPH_RBD_MIRROR_IMAGE_REPLAYER_TYPES_H + +namespace rbd { +namespace mirror { +namespace image_replayer { + +enum HealthState { + HEALTH_STATE_OK, + HEALTH_STATE_WARNING, + HEALTH_STATE_ERROR +}; + +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // CEPH_RBD_MIRROR_IMAGE_REPLAYER_TYPES_H diff --git a/src/tools/rbd_mirror/image_replayer/Utils.cc b/src/tools/rbd_mirror/image_replayer/Utils.cc new file mode 100644 index 000000000..55162a4e4 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/Utils.cc @@ -0,0 +1,61 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "include/rados/librados.hpp" +#include "common/debug.h" +#include "common/errno.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::util::" \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace util { + +std::string compute_image_spec(librados::IoCtx& io_ctx, + const std::string& image_name) { + std::string name = io_ctx.get_namespace(); + if (!name.empty()) { + name += "/"; + } + + return io_ctx.get_pool_name() + "/" + name + image_name; +} + +bool decode_client_meta(const cls::journal::Client& client, + librbd::journal::MirrorPeerClientMeta* client_meta) { + dout(15) << dendl; + + librbd::journal::ClientData client_data; + auto it = client.data.cbegin(); + try { + decode(client_data, it); + } catch (const buffer::error &err) { + derr << "failed to decode client meta data: " << err.what() << dendl; + return false; + } + + auto local_client_meta = boost::get<librbd::journal::MirrorPeerClientMeta>( + &client_data.client_meta); + if (local_client_meta == nullptr) { + derr << "unknown peer registration" << dendl; + return false; + } + + *client_meta = *local_client_meta; + dout(15) << "client found: client_meta=" << *client_meta << dendl; + return true; +} + +} // namespace util +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + diff --git a/src/tools/rbd_mirror/image_replayer/Utils.h b/src/tools/rbd_mirror/image_replayer/Utils.h new file mode 100644 index 000000000..6c5352cd1 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/Utils.h @@ -0,0 +1,29 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_UTILS_H +#define RBD_MIRROR_IMAGE_REPLAYER_UTILS_H + +#include "include/rados/librados_fwd.hpp" +#include <string> + +namespace cls { namespace journal { struct Client; } } +namespace librbd { namespace journal { struct MirrorPeerClientMeta; } } + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace util { + +std::string compute_image_spec(librados::IoCtx& io_ctx, + const std::string& image_name); + +bool decode_client_meta(const cls::journal::Client& client, + librbd::journal::MirrorPeerClientMeta* client_meta); + +} // namespace util +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_REPLAYER_UTILS_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc new file mode 100644 index 000000000..087cf4f5f --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.cc @@ -0,0 +1,162 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "CreateLocalImageRequest.h" +#include "include/rados/librados.hpp" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/journal/Types.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "CreateLocalImageRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; + +template <typename I> +void CreateLocalImageRequest<I>::send() { + unregister_client(); +} + +template <typename I> +void CreateLocalImageRequest<I>::unregister_client() { + dout(10) << dendl; + update_progress("UNREGISTER_CLIENT"); + + auto ctx = create_context_callback< + CreateLocalImageRequest<I>, + &CreateLocalImageRequest<I>::handle_unregister_client>(this); + m_state_builder->remote_journaler->unregister_client(ctx); +} + +template <typename I> +void CreateLocalImageRequest<I>::handle_unregister_client(int r) { + dout(10) << "r=" << r << dendl; + if (r < 0 && r != -ENOENT) { + derr << "failed to unregister with remote journal: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + m_state_builder->local_image_id = ""; + m_state_builder->remote_client_meta = {}; + register_client(); +} + +template <typename I> +void CreateLocalImageRequest<I>::register_client() { + ceph_assert(m_state_builder->local_image_id.empty()); + m_state_builder->local_image_id = + librbd::util::generate_image_id<I>(m_local_io_ctx); + dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl; + update_progress("REGISTER_CLIENT"); + + librbd::journal::MirrorPeerClientMeta client_meta{ + m_state_builder->local_image_id}; + client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING; + + librbd::journal::ClientData client_data{client_meta}; + bufferlist client_data_bl; + encode(client_data, client_data_bl); + + auto ctx = create_context_callback< + CreateLocalImageRequest<I>, + &CreateLocalImageRequest<I>::handle_register_client>(this); + m_state_builder->remote_journaler->register_client(client_data_bl, ctx); +} + +template <typename I> +void CreateLocalImageRequest<I>::handle_register_client(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to register with remote journal: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + m_state_builder->remote_client_state = cls::journal::CLIENT_STATE_CONNECTED; + m_state_builder->remote_client_meta = {m_state_builder->local_image_id}; + m_state_builder->remote_client_meta.state = + librbd::journal::MIRROR_PEER_STATE_SYNCING; + + create_local_image(); +} + +template <typename I> +void CreateLocalImageRequest<I>::create_local_image() { + dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl; + update_progress("CREATE_LOCAL_IMAGE"); + + m_remote_image_ctx->image_lock.lock_shared(); + std::string image_name = m_remote_image_ctx->name; + m_remote_image_ctx->image_lock.unlock_shared(); + + auto ctx = create_context_callback< + CreateLocalImageRequest<I>, + &CreateLocalImageRequest<I>::handle_create_local_image>(this); + auto request = CreateImageRequest<I>::create( + m_threads, m_local_io_ctx, m_global_image_id, + m_state_builder->remote_mirror_uuid, image_name, + m_state_builder->local_image_id, m_remote_image_ctx, + m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_JOURNAL, ctx); + request->send(); +} +template <typename I> +void CreateLocalImageRequest<I>::handle_create_local_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -EBADF) { + dout(5) << "image id " << m_state_builder->local_image_id << " " + << "already in-use" << dendl; + unregister_client(); + return; + } else if (r < 0) { + if (r == -ENOENT) { + dout(10) << "parent image does not exist" << dendl; + } else { + derr << "failed to create local image: " << cpp_strerror(r) << dendl; + } + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void CreateLocalImageRequest<I>::update_progress( + const std::string& description) { + dout(15) << description << dendl; + if (m_progress_ctx != nullptr) { + m_progress_ctx->update_progress(description); + } +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::CreateLocalImageRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h new file mode 100644 index 000000000..fc776ecc3 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h @@ -0,0 +1,116 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_CREATE_LOCAL_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_CREATE_LOCAL_IMAGE_REQUEST_H + +#include "include/rados/librados_fwd.hpp" +#include "tools/rbd_mirror/BaseRequest.h" +#include <string> + +struct Context; +namespace journal { class Journaler; } +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { + +class PoolMetaCache; +class ProgressContext; +template <typename> struct Threads; + +namespace image_replayer { +namespace journal { + +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class CreateLocalImageRequest : public BaseRequest { +public: + typedef rbd::mirror::ProgressContext ProgressContext; + + static CreateLocalImageRequest* create( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + ImageCtxT* remote_image_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + Context* on_finish) { + return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx, + global_image_id, pool_meta_cache, + progress_ctx, state_builder, on_finish); + } + + CreateLocalImageRequest( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + ImageCtxT* remote_image_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + Context* on_finish) + : BaseRequest(on_finish), + m_threads(threads), + m_local_io_ctx(local_io_ctx), + m_remote_image_ctx(remote_image_ctx), + m_global_image_id(global_image_id), + m_pool_meta_cache(pool_meta_cache), + m_progress_ctx(progress_ctx), + m_state_builder(state_builder) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * UNREGISTER_CLIENT < * * * * * * * * + * | * + * v * + * REGISTER_CLIENT * + * | * + * v (id exists) * + * CREATE_LOCAL_IMAGE * * * * * * * * * + * | + * v + * <finish> + * + * @endverbatim + */ + + Threads<ImageCtxT>* m_threads; + librados::IoCtx& m_local_io_ctx; + ImageCtxT* m_remote_image_ctx; + std::string m_global_image_id; + PoolMetaCache* m_pool_meta_cache; + ProgressContext* m_progress_ctx; + StateBuilder<ImageCtxT>* m_state_builder; + + void unregister_client(); + void handle_unregister_client(int r); + + void register_client(); + void handle_register_client(int r); + + void create_local_image(); + void handle_create_local_image(int r); + + void update_progress(const std::string& description); + +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::CreateLocalImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_CREATE_LOCAL_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/EventPreprocessor.cc b/src/tools/rbd_mirror/image_replayer/journal/EventPreprocessor.cc new file mode 100644 index 000000000..f5d49048e --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/EventPreprocessor.cc @@ -0,0 +1,206 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "EventPreprocessor.h" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/journal/Types.h" +#include <boost/variant.hpp> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror + +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "EventPreprocessor: " << this << " " << __func__ \ + << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using librbd::util::create_context_callback; + +template <typename I> +EventPreprocessor<I>::EventPreprocessor(I &local_image_ctx, + Journaler &remote_journaler, + const std::string &local_mirror_uuid, + MirrorPeerClientMeta *client_meta, + librbd::asio::ContextWQ *work_queue) + : m_local_image_ctx(local_image_ctx), m_remote_journaler(remote_journaler), + m_local_mirror_uuid(local_mirror_uuid), m_client_meta(client_meta), + m_work_queue(work_queue) { +} + +template <typename I> +EventPreprocessor<I>::~EventPreprocessor() { + ceph_assert(!m_in_progress); +} + +template <typename I> +bool EventPreprocessor<I>::is_required(const EventEntry &event_entry) { + SnapSeqs snap_seqs(m_client_meta->snap_seqs); + return (prune_snap_map(&snap_seqs) || + event_entry.get_event_type() == + librbd::journal::EVENT_TYPE_SNAP_RENAME); +} + +template <typename I> +void EventPreprocessor<I>::preprocess(EventEntry *event_entry, + Context *on_finish) { + ceph_assert(!m_in_progress); + m_in_progress = true; + m_event_entry = event_entry; + m_on_finish = on_finish; + + refresh_image(); +} + +template <typename I> +void EventPreprocessor<I>::refresh_image() { + dout(20) << dendl; + + Context *ctx = create_context_callback< + EventPreprocessor<I>, &EventPreprocessor<I>::handle_refresh_image>(this); + m_local_image_ctx.state->refresh(ctx); +} + +template <typename I> +void EventPreprocessor<I>::handle_refresh_image(int r) { + dout(20) << "r=" << r << dendl; + + if (r < 0) { + derr << "error encountered during image refresh: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + preprocess_event(); +} + +template <typename I> +void EventPreprocessor<I>::preprocess_event() { + dout(20) << dendl; + + m_snap_seqs = m_client_meta->snap_seqs; + m_snap_seqs_updated = prune_snap_map(&m_snap_seqs); + + int r = boost::apply_visitor(PreprocessEventVisitor(this), + m_event_entry->event); + if (r < 0) { + finish(r); + return; + } + + update_client(); +} + +template <typename I> +int EventPreprocessor<I>::preprocess_snap_rename( + librbd::journal::SnapRenameEvent &event) { + dout(20) << "remote_snap_id=" << event.snap_id << ", " + << "src_snap_name=" << event.src_snap_name << ", " + << "dest_snap_name=" << event.dst_snap_name << dendl; + + auto snap_seq_it = m_snap_seqs.find(event.snap_id); + if (snap_seq_it != m_snap_seqs.end()) { + dout(20) << "remapping remote snap id " << snap_seq_it->first << " " + << "to local snap id " << snap_seq_it->second << dendl; + event.snap_id = snap_seq_it->second; + return 0; + } + + auto snap_id_it = m_local_image_ctx.snap_ids.find({cls::rbd::UserSnapshotNamespace(), + event.src_snap_name}); + if (snap_id_it == m_local_image_ctx.snap_ids.end()) { + dout(20) << "cannot map remote snapshot '" << event.src_snap_name << "' " + << "to local snapshot" << dendl; + event.snap_id = CEPH_NOSNAP; + return -ENOENT; + } + + dout(20) << "mapping remote snap id " << event.snap_id << " " + << "to local snap id " << snap_id_it->second << dendl; + m_snap_seqs_updated = true; + m_snap_seqs[event.snap_id] = snap_id_it->second; + event.snap_id = snap_id_it->second; + return 0; +} + +template <typename I> +void EventPreprocessor<I>::update_client() { + if (!m_snap_seqs_updated) { + finish(0); + return; + } + + dout(20) << dendl; + librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta); + client_meta.snap_seqs = m_snap_seqs; + + librbd::journal::ClientData client_data(client_meta); + bufferlist data_bl; + encode(client_data, data_bl); + + Context *ctx = create_context_callback< + EventPreprocessor<I>, &EventPreprocessor<I>::handle_update_client>( + this); + m_remote_journaler.update_client(data_bl, ctx); +} + +template <typename I> +void EventPreprocessor<I>::handle_update_client(int r) { + dout(20) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to update mirror peer journal client: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_client_meta->snap_seqs = m_snap_seqs; + finish(0); +} + +template <typename I> +bool EventPreprocessor<I>::prune_snap_map(SnapSeqs *snap_seqs) { + bool pruned = false; + + std::shared_lock image_locker{m_local_image_ctx.image_lock}; + for (auto it = snap_seqs->begin(); it != snap_seqs->end(); ) { + auto current_it(it++); + if (m_local_image_ctx.snap_info.count(current_it->second) == 0) { + snap_seqs->erase(current_it); + pruned = true; + } + } + return pruned; +} + +template <typename I> +void EventPreprocessor<I>::finish(int r) { + dout(20) << "r=" << r << dendl; + + Context *on_finish = m_on_finish; + m_on_finish = nullptr; + m_event_entry = nullptr; + m_in_progress = false; + m_snap_seqs_updated = false; + m_work_queue->queue(on_finish, r); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::EventPreprocessor<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/journal/EventPreprocessor.h b/src/tools/rbd_mirror/image_replayer/journal/EventPreprocessor.h new file mode 100644 index 000000000..12f70eb93 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/EventPreprocessor.h @@ -0,0 +1,127 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_EVENT_PREPROCESSOR_H +#define RBD_MIRROR_IMAGE_REPLAYER_EVENT_PREPROCESSOR_H + +#include "include/int_types.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include <map> +#include <string> +#include <boost/variant/static_visitor.hpp> + +struct Context; +namespace journal { class Journaler; } +namespace librbd { +class ImageCtx; +namespace asio { struct ContextWQ; } +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template <typename ImageCtxT = librbd::ImageCtx> +class EventPreprocessor { +public: + using Journaler = typename librbd::journal::TypeTraits<ImageCtxT>::Journaler; + using EventEntry = librbd::journal::EventEntry; + using MirrorPeerClientMeta = librbd::journal::MirrorPeerClientMeta; + + static EventPreprocessor *create(ImageCtxT &local_image_ctx, + Journaler &remote_journaler, + const std::string &local_mirror_uuid, + MirrorPeerClientMeta *client_meta, + librbd::asio::ContextWQ *work_queue) { + return new EventPreprocessor(local_image_ctx, remote_journaler, + local_mirror_uuid, client_meta, work_queue); + } + + static void destroy(EventPreprocessor* processor) { + delete processor; + } + + EventPreprocessor(ImageCtxT &local_image_ctx, Journaler &remote_journaler, + const std::string &local_mirror_uuid, + MirrorPeerClientMeta *client_meta, + librbd::asio::ContextWQ *work_queue); + ~EventPreprocessor(); + + bool is_required(const EventEntry &event_entry); + void preprocess(EventEntry *event_entry, Context *on_finish); + +private: + /** + * @verbatim + * + * <start> + * | + * v (skip if not required) + * REFRESH_IMAGE + * | + * v (skip if not required) + * PREPROCESS_EVENT + * | + * v (skip if not required) + * UPDATE_CLIENT + * + * @endverbatim + */ + + typedef std::map<uint64_t, uint64_t> SnapSeqs; + + class PreprocessEventVisitor : public boost::static_visitor<int> { + public: + EventPreprocessor *event_preprocessor; + + PreprocessEventVisitor(EventPreprocessor *event_preprocessor) + : event_preprocessor(event_preprocessor) { + } + + template <typename T> + inline int operator()(T&) const { + return 0; + } + inline int operator()(librbd::journal::SnapRenameEvent &event) const { + return event_preprocessor->preprocess_snap_rename(event); + } + }; + + ImageCtxT &m_local_image_ctx; + Journaler &m_remote_journaler; + std::string m_local_mirror_uuid; + MirrorPeerClientMeta *m_client_meta; + librbd::asio::ContextWQ *m_work_queue; + + bool m_in_progress = false; + EventEntry *m_event_entry = nullptr; + Context *m_on_finish = nullptr; + + SnapSeqs m_snap_seqs; + bool m_snap_seqs_updated = false; + + bool prune_snap_map(SnapSeqs *snap_seqs); + + void refresh_image(); + void handle_refresh_image(int r); + + void preprocess_event(); + int preprocess_snap_rename(librbd::journal::SnapRenameEvent &event); + + void update_client(); + void handle_update_client(int r); + + void finish(int r); + +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::EventPreprocessor<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_EVENT_PREPROCESSOR_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc new file mode 100644 index 000000000..c8a96a4ad --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.cc @@ -0,0 +1,316 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "PrepareReplayRequest.h" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "PrepareReplayRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using librbd::util::create_context_callback; + +template <typename I> +void PrepareReplayRequest<I>::send() { + *m_resync_requested = false; + *m_syncing = false; + + if (m_state_builder->local_image_id != + m_state_builder->remote_client_meta.image_id) { + // somehow our local image has a different image id than the image id + // registered in the remote image + derr << "split-brain detected: local_image_id=" + << m_state_builder->local_image_id << ", " + << "registered local_image_id=" + << m_state_builder->remote_client_meta.image_id << dendl; + finish(-EEXIST); + return; + } + + std::shared_lock image_locker(m_state_builder->local_image_ctx->image_lock); + if (m_state_builder->local_image_ctx->journal == nullptr) { + image_locker.unlock(); + + derr << "local image does not support journaling" << dendl; + finish(-EINVAL); + return; + } + + int r = m_state_builder->local_image_ctx->journal->is_resync_requested( + m_resync_requested); + if (r < 0) { + image_locker.unlock(); + + derr << "failed to check if a resync was requested" << dendl; + finish(r); + return; + } + + m_local_tag_tid = m_state_builder->local_image_ctx->journal->get_tag_tid(); + m_local_tag_data = m_state_builder->local_image_ctx->journal->get_tag_data(); + dout(10) << "local tag=" << m_local_tag_tid << ", " + << "local tag data=" << m_local_tag_data << dendl; + image_locker.unlock(); + + if (*m_resync_requested) { + finish(0); + return; + } else if (m_state_builder->remote_client_meta.state == + librbd::journal::MIRROR_PEER_STATE_SYNCING && + m_local_tag_data.mirror_uuid == + m_state_builder->remote_mirror_uuid) { + // if the initial sync hasn't completed, we cannot replay + *m_syncing = true; + finish(0); + return; + } + + update_client_state(); +} + +template <typename I> +void PrepareReplayRequest<I>::update_client_state() { + if (m_state_builder->remote_client_meta.state != + librbd::journal::MIRROR_PEER_STATE_SYNCING || + m_local_tag_data.mirror_uuid == m_state_builder->remote_mirror_uuid) { + get_remote_tag_class(); + return; + } + + // our local image is not primary, is flagged as syncing on the remote side, + // but is no longer tied to the remote -- this implies we were forced + // promoted and then demoted at some point + dout(15) << dendl; + update_progress("UPDATE_CLIENT_STATE"); + + auto client_meta = m_state_builder->remote_client_meta; + client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + + librbd::journal::ClientData client_data(client_meta); + bufferlist data_bl; + encode(client_data, data_bl); + + auto ctx = create_context_callback< + PrepareReplayRequest<I>, + &PrepareReplayRequest<I>::handle_update_client_state>(this); + m_state_builder->remote_journaler->update_client(data_bl, ctx); +} + +template <typename I> +void PrepareReplayRequest<I>::handle_update_client_state(int r) { + dout(15) << "r=" << r << dendl; + if (r < 0) { + derr << "failed to update client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_state_builder->remote_client_meta.state = + librbd::journal::MIRROR_PEER_STATE_REPLAYING; + get_remote_tag_class(); +} + +template <typename I> +void PrepareReplayRequest<I>::get_remote_tag_class() { + dout(10) << dendl; + update_progress("GET_REMOTE_TAG_CLASS"); + + auto ctx = create_context_callback< + PrepareReplayRequest<I>, + &PrepareReplayRequest<I>::handle_get_remote_tag_class>(this); + m_state_builder->remote_journaler->get_client( + librbd::Journal<>::IMAGE_CLIENT_ID, &m_client, ctx); +} + +template <typename I> +void PrepareReplayRequest<I>::handle_get_remote_tag_class(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to retrieve remote client: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + librbd::journal::ClientData client_data; + auto it = m_client.data.cbegin(); + try { + decode(client_data, it); + } catch (const buffer::error &err) { + derr << "failed to decode remote client meta data: " << err.what() + << dendl; + finish(-EBADMSG); + return; + } + + librbd::journal::ImageClientMeta *client_meta = + boost::get<librbd::journal::ImageClientMeta>(&client_data.client_meta); + if (client_meta == nullptr) { + derr << "unknown remote client registration" << dendl; + finish(-EINVAL); + return; + } + + m_remote_tag_class = client_meta->tag_class; + dout(10) << "remote tag class=" << m_remote_tag_class << dendl; + + get_remote_tags(); +} + +template <typename I> +void PrepareReplayRequest<I>::get_remote_tags() { + dout(10) << dendl; + update_progress("GET_REMOTE_TAGS"); + + auto ctx = create_context_callback< + PrepareReplayRequest<I>, + &PrepareReplayRequest<I>::handle_get_remote_tags>(this); + m_state_builder->remote_journaler->get_tags(m_remote_tag_class, + &m_remote_tags, ctx); +} + +template <typename I> +void PrepareReplayRequest<I>::handle_get_remote_tags(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to retrieve remote tags: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + // At this point, the local image was existing, non-primary, and replaying; + // and the remote image is primary. Attempt to link the local image's most + // recent tag to the remote image's tag chain. + bool remote_tag_data_valid = false; + librbd::journal::TagData remote_tag_data; + boost::optional<uint64_t> remote_orphan_tag_tid = + boost::make_optional<uint64_t>(false, 0U); + bool reconnect_orphan = false; + + // decode the remote tags + for (auto &remote_tag : m_remote_tags) { + if (m_local_tag_data.predecessor.commit_valid && + m_local_tag_data.predecessor.mirror_uuid == + m_state_builder->remote_mirror_uuid && + m_local_tag_data.predecessor.tag_tid > remote_tag.tid) { + dout(10) << "skipping processed predecessor remote tag " + << remote_tag.tid << dendl; + continue; + } + + try { + auto it = remote_tag.data.cbegin(); + decode(remote_tag_data, it); + remote_tag_data_valid = true; + } catch (const buffer::error &err) { + derr << "failed to decode remote tag " << remote_tag.tid << ": " + << err.what() << dendl; + finish(-EBADMSG); + return; + } + + dout(10) << "decoded remote tag " << remote_tag.tid << ": " + << remote_tag_data << dendl; + + if (!m_local_tag_data.predecessor.commit_valid) { + // newly synced local image (no predecessor) replays from the first tag + if (remote_tag_data.mirror_uuid != librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(10) << "skipping non-primary remote tag" << dendl; + continue; + } + + dout(10) << "using initial primary remote tag" << dendl; + break; + } + + if (m_local_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID) { + // demotion last available local epoch + + if (remote_tag_data.mirror_uuid == m_local_tag_data.mirror_uuid && + remote_tag_data.predecessor.commit_valid && + remote_tag_data.predecessor.tag_tid == + m_local_tag_data.predecessor.tag_tid) { + // demotion matches remote epoch + + if (remote_tag_data.predecessor.mirror_uuid == m_local_mirror_uuid && + m_local_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::LOCAL_MIRROR_UUID) { + // local demoted and remote has matching event + dout(10) << "found matching local demotion tag" << dendl; + remote_orphan_tag_tid = remote_tag.tid; + continue; + } + + if (m_local_tag_data.predecessor.mirror_uuid == + m_state_builder->remote_mirror_uuid && + remote_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::LOCAL_MIRROR_UUID) { + // remote demoted and local has matching event + dout(10) << "found matching remote demotion tag" << dendl; + remote_orphan_tag_tid = remote_tag.tid; + continue; + } + } + + if (remote_tag_data.mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID && + remote_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::ORPHAN_MIRROR_UUID && + remote_tag_data.predecessor.commit_valid && remote_orphan_tag_tid && + remote_tag_data.predecessor.tag_tid == *remote_orphan_tag_tid) { + // remote promotion tag chained to remote/local demotion tag + dout(10) << "found chained remote promotion tag" << dendl; + reconnect_orphan = true; + break; + } + + // promotion must follow demotion + remote_orphan_tag_tid = boost::none; + } + } + + if (remote_tag_data_valid && + m_local_tag_data.mirror_uuid == m_state_builder->remote_mirror_uuid) { + dout(10) << "local image is in clean replay state" << dendl; + } else if (reconnect_orphan) { + dout(10) << "remote image was demoted/promoted" << dendl; + } else { + derr << "split-brain detected -- skipping image replay" << dendl; + finish(-EEXIST); + return; + } + + finish(0); +} + +template <typename I> +void PrepareReplayRequest<I>::update_progress(const std::string &description) { + dout(10) << description << dendl; + + if (m_progress_ctx != nullptr) { + m_progress_ctx->update_progress(description); + } +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::PrepareReplayRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h new file mode 100644 index 000000000..2b6fb659b --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h @@ -0,0 +1,115 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H + +#include "include/int_types.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Types.h" +#include "librbd/mirror/Types.h" +#include "tools/rbd_mirror/BaseRequest.h" +#include <list> +#include <string> + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { + +class ProgressContext; + +namespace image_replayer { +namespace journal { + +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class PrepareReplayRequest : public BaseRequest { +public: + static PrepareReplayRequest* create( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + bool* resync_requested, + bool* syncing, + Context* on_finish) { + return new PrepareReplayRequest( + local_mirror_uuid, progress_ctx, state_builder, resync_requested, + syncing, on_finish); + } + + PrepareReplayRequest( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + bool* resync_requested, + bool* syncing, + Context* on_finish) + : BaseRequest(on_finish), + m_local_mirror_uuid(local_mirror_uuid), + m_progress_ctx(progress_ctx), + m_state_builder(state_builder), + m_resync_requested(resync_requested), + m_syncing(syncing) { + } + + void send() override; + +private: + /** + * @verbatim + * + * <start> + * | + * v + * UPDATE_CLIENT_STATE + * | + * v + * GET_REMOTE_TAG_CLASS + * | + * v + * GET_REMOTE_TAGS + * | + * v + * <finish> + * + * @endverbatim + */ + typedef std::list<cls::journal::Tag> Tags; + + std::string m_local_mirror_uuid; + ProgressContext* m_progress_ctx; + StateBuilder<ImageCtxT>* m_state_builder; + bool* m_resync_requested; + bool* m_syncing; + + uint64_t m_local_tag_tid = 0; + librbd::journal::TagData m_local_tag_data; + + uint64_t m_remote_tag_class = 0; + Tags m_remote_tags; + cls::journal::Client m_client; + + void update_client_state(); + void handle_update_client_state(int r); + + void get_remote_tag_class(); + void handle_get_remote_tag_class(int r); + + void get_remote_tags(); + void handle_get_remote_tags(int r); + + void update_progress(const std::string& description); + +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::PrepareReplayRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc new file mode 100644 index 000000000..eb99d5add --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.cc @@ -0,0 +1,284 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "ReplayStatusFormatter.h" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "json_spirit/json_spirit.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "ReplayStatusFormatter: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +using librbd::util::unique_lock_name; + +namespace { + +double round_to_two_places(double value) { + return abs(round(value * 100) / 100); +} + +json_spirit::mObject to_json_object( + const cls::journal::ObjectPosition& position) { + json_spirit::mObject object; + if (position != cls::journal::ObjectPosition{}) { + object["object_number"] = position.object_number; + object["tag_tid"] = position.tag_tid; + object["entry_tid"] = position.entry_tid; + } + return object; +} + +} // anonymous namespace + +template <typename I> +ReplayStatusFormatter<I>::ReplayStatusFormatter(Journaler *journaler, + const std::string &mirror_uuid) + : m_journaler(journaler), + m_mirror_uuid(mirror_uuid), + m_lock(ceph::make_mutex(unique_lock_name("ReplayStatusFormatter::m_lock", this))) { +} + +template <typename I> +void ReplayStatusFormatter<I>::handle_entry_processed(uint32_t bytes) { + dout(20) << dendl; + + m_bytes_per_second(bytes); + m_entries_per_second(1); +} + +template <typename I> +bool ReplayStatusFormatter<I>::get_or_send_update(std::string *description, + Context *on_finish) { + dout(20) << dendl; + + bool in_progress = false; + { + std::lock_guard locker{m_lock}; + if (m_on_finish) { + in_progress = true; + } else { + m_on_finish = on_finish; + } + } + + if (in_progress) { + dout(10) << "previous request is still in progress, ignoring" << dendl; + on_finish->complete(-EAGAIN); + return false; + } + + m_master_position = cls::journal::ObjectPosition(); + m_mirror_position = cls::journal::ObjectPosition(); + + cls::journal::Client master_client, mirror_client; + int r; + + r = m_journaler->get_cached_client(librbd::Journal<>::IMAGE_CLIENT_ID, + &master_client); + if (r < 0) { + derr << "error retrieving registered master client: " + << cpp_strerror(r) << dendl; + } else { + r = m_journaler->get_cached_client(m_mirror_uuid, &mirror_client); + if (r < 0) { + derr << "error retrieving registered mirror client: " + << cpp_strerror(r) << dendl; + } + } + + if (!master_client.commit_position.object_positions.empty()) { + m_master_position = + *(master_client.commit_position.object_positions.begin()); + } + + if (!mirror_client.commit_position.object_positions.empty()) { + m_mirror_position = + *(mirror_client.commit_position.object_positions.begin()); + } + + if (!calculate_behind_master_or_send_update()) { + dout(20) << "need to update tag cache" << dendl; + return false; + } + + format(description); + + { + std::lock_guard locker{m_lock}; + ceph_assert(m_on_finish == on_finish); + m_on_finish = nullptr; + } + + on_finish->complete(-EEXIST); + return true; +} + +template <typename I> +bool ReplayStatusFormatter<I>::calculate_behind_master_or_send_update() { + dout(20) << "m_master_position=" << m_master_position + << ", m_mirror_position=" << m_mirror_position << dendl; + + m_entries_behind_master = 0; + + if (m_master_position == cls::journal::ObjectPosition() || + m_master_position.tag_tid < m_mirror_position.tag_tid) { + return true; + } + + cls::journal::ObjectPosition master = m_master_position; + uint64_t mirror_tag_tid = m_mirror_position.tag_tid; + + while (master.tag_tid > mirror_tag_tid) { + auto tag_it = m_tag_cache.find(master.tag_tid); + if (tag_it == m_tag_cache.end()) { + send_update_tag_cache(master.tag_tid, mirror_tag_tid); + return false; + } + librbd::journal::TagData &tag_data = tag_it->second; + m_entries_behind_master += master.entry_tid; + master = {0, tag_data.predecessor.tag_tid, tag_data.predecessor.entry_tid}; + } + if (master.tag_tid == mirror_tag_tid && + master.entry_tid > m_mirror_position.entry_tid) { + m_entries_behind_master += master.entry_tid - m_mirror_position.entry_tid; + } + + dout(20) << "clearing tags not needed any more (below mirror position)" + << dendl; + + uint64_t tag_tid = mirror_tag_tid; + size_t old_size = m_tag_cache.size(); + while (tag_tid != 0) { + auto tag_it = m_tag_cache.find(tag_tid); + if (tag_it == m_tag_cache.end()) { + break; + } + librbd::journal::TagData &tag_data = tag_it->second; + + dout(20) << "erasing tag " << tag_data << "for tag_tid " << tag_tid + << dendl; + + tag_tid = tag_data.predecessor.tag_tid; + m_tag_cache.erase(tag_it); + } + + dout(20) << old_size - m_tag_cache.size() << " entries cleared" << dendl; + + return true; +} + +template <typename I> +void ReplayStatusFormatter<I>::send_update_tag_cache(uint64_t master_tag_tid, + uint64_t mirror_tag_tid) { + if (master_tag_tid <= mirror_tag_tid || + m_tag_cache.find(master_tag_tid) != m_tag_cache.end()) { + Context *on_finish = nullptr; + { + std::lock_guard locker{m_lock}; + std::swap(m_on_finish, on_finish); + } + + ceph_assert(on_finish); + on_finish->complete(0); + return; + } + + dout(20) << "master_tag_tid=" << master_tag_tid << ", mirror_tag_tid=" + << mirror_tag_tid << dendl; + + auto ctx = new LambdaContext( + [this, master_tag_tid, mirror_tag_tid](int r) { + handle_update_tag_cache(master_tag_tid, mirror_tag_tid, r); + }); + m_journaler->get_tag(master_tag_tid, &m_tag, ctx); +} + +template <typename I> +void ReplayStatusFormatter<I>::handle_update_tag_cache(uint64_t master_tag_tid, + uint64_t mirror_tag_tid, + int r) { + librbd::journal::TagData tag_data; + + if (r < 0) { + derr << "error retrieving tag " << master_tag_tid << ": " << cpp_strerror(r) + << dendl; + } else { + dout(20) << "retrieved tag " << master_tag_tid << ": " << m_tag << dendl; + + auto it = m_tag.data.cbegin(); + try { + decode(tag_data, it); + } catch (const buffer::error &err) { + derr << "error decoding tag " << master_tag_tid << ": " << err.what() + << dendl; + } + } + + if (tag_data.predecessor.mirror_uuid != + librbd::Journal<>::LOCAL_MIRROR_UUID && + tag_data.predecessor.mirror_uuid != + librbd::Journal<>::ORPHAN_MIRROR_UUID) { + dout(20) << "hit remote image non-primary epoch" << dendl; + tag_data.predecessor = {}; + } + + dout(20) << "decoded tag " << master_tag_tid << ": " << tag_data << dendl; + + m_tag_cache[master_tag_tid] = tag_data; + send_update_tag_cache(tag_data.predecessor.tag_tid, mirror_tag_tid); +} + +template <typename I> +void ReplayStatusFormatter<I>::format(std::string *description) { + dout(20) << "m_master_position=" << m_master_position + << ", m_mirror_position=" << m_mirror_position + << ", m_entries_behind_master=" << m_entries_behind_master << dendl; + + json_spirit::mObject root_obj; + root_obj["primary_position"] = to_json_object(m_master_position); + root_obj["non_primary_position"] = to_json_object(m_mirror_position); + root_obj["entries_behind_primary"] = ( + m_entries_behind_master > 0 ? m_entries_behind_master : 0); + + m_bytes_per_second(0); + root_obj["bytes_per_second"] = round_to_two_places( + m_bytes_per_second.get_average()); + + m_entries_per_second(0); + auto entries_per_second = m_entries_per_second.get_average(); + root_obj["entries_per_second"] = round_to_two_places(entries_per_second); + + if (m_entries_behind_master > 0 && entries_per_second > 0) { + std::uint64_t seconds_until_synced = round_to_two_places( + m_entries_behind_master / entries_per_second); + if (seconds_until_synced >= std::numeric_limits<uint64_t>::max()) { + seconds_until_synced = std::numeric_limits<uint64_t>::max(); + } + + root_obj["seconds_until_synced"] = seconds_until_synced; + } + + *description = json_spirit::write( + root_obj, json_spirit::remove_trailing_zeros); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::ReplayStatusFormatter<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h new file mode 100644 index 000000000..5dbbfe10d --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h @@ -0,0 +1,70 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_REPLAY_STATUS_FORMATTER_H +#define RBD_MIRROR_IMAGE_REPLAYER_REPLAY_STATUS_FORMATTER_H + +#include "include/Context.h" +#include "common/ceph_mutex.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h" + +namespace journal { class Journaler; } +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template <typename ImageCtxT = librbd::ImageCtx> +class ReplayStatusFormatter { +public: + typedef typename librbd::journal::TypeTraits<ImageCtxT>::Journaler Journaler; + + static ReplayStatusFormatter* create(Journaler *journaler, + const std::string &mirror_uuid) { + return new ReplayStatusFormatter(journaler, mirror_uuid); + } + + static void destroy(ReplayStatusFormatter* formatter) { + delete formatter; + } + + ReplayStatusFormatter(Journaler *journaler, const std::string &mirror_uuid); + + void handle_entry_processed(uint32_t bytes); + + bool get_or_send_update(std::string *description, Context *on_finish); + +private: + Journaler *m_journaler; + std::string m_mirror_uuid; + ceph::mutex m_lock; + Context *m_on_finish = nullptr; + cls::journal::ObjectPosition m_master_position; + cls::journal::ObjectPosition m_mirror_position; + int64_t m_entries_behind_master = 0; + cls::journal::Tag m_tag; + std::map<uint64_t, librbd::journal::TagData> m_tag_cache; + + TimeRollingMean m_bytes_per_second; + TimeRollingMean m_entries_per_second; + + bool calculate_behind_master_or_send_update(); + void send_update_tag_cache(uint64_t master_tag_tid, uint64_t mirror_tag_tid); + void handle_update_tag_cache(uint64_t master_tag_tid, uint64_t mirror_tag_tid, + int r); + void format(std::string *description); +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::ReplayStatusFormatter<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_REPLAY_STATUS_FORMATTER_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc b/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc new file mode 100644 index 000000000..3ce9104d2 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/Replayer.cc @@ -0,0 +1,1303 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "Replayer.h" +#include "common/debug.h" +#include "common/errno.h" +#include "common/Timer.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/journal/Replay.h" +#include "journal/Journaler.h" +#include "journal/JournalMetadataListener.h" +#include "journal/ReplayHandler.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/ReplayerListener.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_replayer/journal/EventPreprocessor.h" +#include "tools/rbd_mirror/image_replayer/journal/ReplayStatusFormatter.h" +#include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "Replayer: " << this << " " << __func__ << ": " + +extern PerfCounters *g_journal_perf_counters; + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +namespace { + +uint32_t calculate_replay_delay(const utime_t &event_time, + int mirroring_replay_delay) { + if (mirroring_replay_delay <= 0) { + return 0; + } + + utime_t now = ceph_clock_now(); + if (event_time + mirroring_replay_delay <= now) { + return 0; + } + + // ensure it is rounded up when converting to integer + return (event_time + mirroring_replay_delay - now) + 1; +} + +} // anonymous namespace + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; + +template <typename I> +struct Replayer<I>::C_ReplayCommitted : public Context { + Replayer* replayer; + ReplayEntry replay_entry; + uint64_t replay_bytes; + utime_t replay_start_time; + + C_ReplayCommitted(Replayer* replayer, ReplayEntry &&replay_entry, + uint64_t replay_bytes, const utime_t &replay_start_time) + : replayer(replayer), replay_entry(std::move(replay_entry)), + replay_bytes(replay_bytes), replay_start_time(replay_start_time) { + } + + void finish(int r) override { + replayer->handle_process_entry_safe(replay_entry, replay_bytes, + replay_start_time, r); + } +}; + +template <typename I> +struct Replayer<I>::RemoteJournalerListener + : public ::journal::JournalMetadataListener { + Replayer* replayer; + + RemoteJournalerListener(Replayer* replayer) : replayer(replayer) {} + + void handle_update(::journal::JournalMetadata*) override { + auto ctx = new C_TrackedOp( + replayer->m_in_flight_op_tracker, + new LambdaContext([this](int r) { + replayer->handle_remote_journal_metadata_updated(); + })); + replayer->m_threads->work_queue->queue(ctx, 0); + } +}; + +template <typename I> +struct Replayer<I>::RemoteReplayHandler : public ::journal::ReplayHandler { + Replayer* replayer; + + RemoteReplayHandler(Replayer* replayer) : replayer(replayer) {} + ~RemoteReplayHandler() override {}; + + void handle_entries_available() override { + replayer->handle_replay_ready(); + } + + void handle_complete(int r) override { + std::string error; + if (r == -ENOMEM) { + error = "not enough memory in autotune cache"; + } else if (r < 0) { + error = "replay completed with error: " + cpp_strerror(r); + } + replayer->handle_replay_complete(r, error); + } +}; + +template <typename I> +struct Replayer<I>::LocalJournalListener + : public librbd::journal::Listener { + Replayer* replayer; + + LocalJournalListener(Replayer* replayer) : replayer(replayer) { + } + + void handle_close() override { + replayer->handle_replay_complete(0, ""); + } + + void handle_promoted() override { + replayer->handle_replay_complete(0, "force promoted"); + } + + void handle_resync() override { + replayer->handle_resync_image(); + } +}; + +template <typename I> +Replayer<I>::Replayer( + Threads<I>* threads, + const std::string& local_mirror_uuid, + StateBuilder<I>* state_builder, + ReplayerListener* replayer_listener) + : m_threads(threads), + m_local_mirror_uuid(local_mirror_uuid), + m_state_builder(state_builder), + m_replayer_listener(replayer_listener), + m_lock(ceph::make_mutex(librbd::util::unique_lock_name( + "rbd::mirror::image_replayer::journal::Replayer", this))) { + dout(10) << dendl; +} + +template <typename I> +Replayer<I>::~Replayer() { + dout(10) << dendl; + + { + std::unique_lock locker{m_lock}; + unregister_perf_counters(); + } + + ceph_assert(m_remote_listener == nullptr); + ceph_assert(m_local_journal_listener == nullptr); + ceph_assert(m_local_journal_replay == nullptr); + ceph_assert(m_remote_replay_handler == nullptr); + ceph_assert(m_event_preprocessor == nullptr); + ceph_assert(m_replay_status_formatter == nullptr); + ceph_assert(m_delayed_preprocess_task == nullptr); + ceph_assert(m_flush_local_replay_task == nullptr); + ceph_assert(m_state_builder->local_image_ctx == nullptr); +} + +template <typename I> +void Replayer<I>::init(Context* on_finish) { + dout(10) << dendl; + + { + auto local_image_ctx = m_state_builder->local_image_ctx; + std::shared_lock image_locker{local_image_ctx->image_lock}; + m_image_spec = util::compute_image_spec(local_image_ctx->md_ctx, + local_image_ctx->name); + } + + { + std::unique_lock locker{m_lock}; + register_perf_counters(); + } + + ceph_assert(m_on_init_shutdown == nullptr); + m_on_init_shutdown = on_finish; + + init_remote_journaler(); +} + +template <typename I> +void Replayer<I>::shut_down(Context* on_finish) { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + ceph_assert(m_on_init_shutdown == nullptr); + m_on_init_shutdown = on_finish; + + if (m_state == STATE_INIT) { + // raced with the last piece of the init state machine + return; + } else if (m_state == STATE_REPLAYING) { + m_state = STATE_COMPLETE; + } + + // if shutting down due to an error notification, we don't + // need to propagate the same error again + m_error_code = 0; + m_error_description = ""; + + cancel_delayed_preprocess_task(); + cancel_flush_local_replay_task(); + wait_for_flush(); +} + +template <typename I> +void Replayer<I>::flush(Context* on_finish) { + dout(10) << dendl; + + flush_local_replay(new C_TrackedOp(m_in_flight_op_tracker, on_finish)); +} + +template <typename I> +bool Replayer<I>::get_replay_status(std::string* description, + Context* on_finish) { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + if (m_replay_status_formatter == nullptr) { + derr << "replay not running" << dendl; + locker.unlock(); + + on_finish->complete(-EAGAIN); + return false; + } + + on_finish = new C_TrackedOp(m_in_flight_op_tracker, on_finish); + return m_replay_status_formatter->get_or_send_update(description, + on_finish); +} + +template <typename I> +void Replayer<I>::init_remote_journaler() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + Replayer, &Replayer<I>::handle_init_remote_journaler>(this); + m_state_builder->remote_journaler->init(ctx); +} + +template <typename I> +void Replayer<I>::handle_init_remote_journaler(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + if (r < 0) { + derr << "failed to initialize remote journal: " << cpp_strerror(r) << dendl; + handle_replay_complete(locker, r, "error initializing remote journal"); + close_local_image(); + return; + } + + // listen for metadata updates to check for disconnect events + ceph_assert(m_remote_listener == nullptr); + m_remote_listener = new RemoteJournalerListener(this); + m_state_builder->remote_journaler->add_listener(m_remote_listener); + + cls::journal::Client remote_client; + r = m_state_builder->remote_journaler->get_cached_client(m_local_mirror_uuid, + &remote_client); + if (r < 0) { + derr << "error retrieving remote journal client: " << cpp_strerror(r) + << dendl; + handle_replay_complete(locker, r, "error retrieving remote journal client"); + close_local_image(); + return; + } + + std::string error; + r = validate_remote_client_state(remote_client, + &m_state_builder->remote_client_meta, + &m_resync_requested, &error); + if (r < 0) { + handle_replay_complete(locker, r, error); + close_local_image(); + return; + } + + start_external_replay(locker); +} + +template <typename I> +void Replayer<I>::start_external_replay(std::unique_lock<ceph::mutex>& locker) { + dout(10) << dendl; + + auto local_image_ctx = m_state_builder->local_image_ctx; + std::shared_lock local_image_locker{local_image_ctx->image_lock}; + + ceph_assert(m_local_journal == nullptr); + m_local_journal = local_image_ctx->journal; + if (m_local_journal == nullptr) { + local_image_locker.unlock(); + + derr << "local image journal closed" << dendl; + handle_replay_complete(locker, -EINVAL, "error accessing local journal"); + close_local_image(); + return; + } + + // safe to hold pointer to journal after external playback starts + Context *start_ctx = create_context_callback< + Replayer, &Replayer<I>::handle_start_external_replay>(this); + m_local_journal->start_external_replay(&m_local_journal_replay, start_ctx); +} + +template <typename I> +void Replayer<I>::handle_start_external_replay(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + if (r < 0) { + ceph_assert(m_local_journal_replay == nullptr); + derr << "error starting external replay on local image " + << m_state_builder->local_image_ctx->id << ": " + << cpp_strerror(r) << dendl; + + handle_replay_complete(locker, r, "error starting replay on local image"); + close_local_image(); + return; + } + + if (!notify_init_complete(locker)) { + return; + } + + m_state = STATE_REPLAYING; + + // check for resync/promotion state after adding listener + if (!add_local_journal_listener(locker)) { + return; + } + + // start remote journal replay + m_event_preprocessor = EventPreprocessor<I>::create( + *m_state_builder->local_image_ctx, *m_state_builder->remote_journaler, + m_local_mirror_uuid, &m_state_builder->remote_client_meta, + m_threads->work_queue); + m_replay_status_formatter = ReplayStatusFormatter<I>::create( + m_state_builder->remote_journaler, m_local_mirror_uuid); + + auto cct = static_cast<CephContext *>(m_state_builder->local_image_ctx->cct); + double poll_seconds = cct->_conf.get_val<double>( + "rbd_mirror_journal_poll_age"); + m_remote_replay_handler = new RemoteReplayHandler(this); + m_state_builder->remote_journaler->start_live_replay(m_remote_replay_handler, + poll_seconds); + + notify_status_updated(); +} + +template <typename I> +bool Replayer<I>::add_local_journal_listener( + std::unique_lock<ceph::mutex>& locker) { + dout(10) << dendl; + + // listen for promotion and resync requests against local journal + ceph_assert(m_local_journal_listener == nullptr); + m_local_journal_listener = new LocalJournalListener(this); + m_local_journal->add_listener(m_local_journal_listener); + + // verify that the local image wasn't force-promoted and that a resync hasn't + // been requested now that we are listening for events + if (m_local_journal->is_tag_owner()) { + dout(10) << "local image force-promoted" << dendl; + handle_replay_complete(locker, 0, "force promoted"); + return false; + } + + bool resync_requested = false; + int r = m_local_journal->is_resync_requested(&resync_requested); + if (r < 0) { + dout(10) << "failed to determine resync state: " << cpp_strerror(r) + << dendl; + handle_replay_complete(locker, r, "error parsing resync state"); + return false; + } else if (resync_requested) { + dout(10) << "local image resync requested" << dendl; + handle_replay_complete(locker, 0, "resync requested"); + return false; + } + + return true; +} + +template <typename I> +bool Replayer<I>::notify_init_complete(std::unique_lock<ceph::mutex>& locker) { + dout(10) << dendl; + + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + ceph_assert(m_state == STATE_INIT); + + // notify that init has completed + Context *on_finish = nullptr; + std::swap(m_on_init_shutdown, on_finish); + + locker.unlock(); + on_finish->complete(0); + locker.lock(); + + if (m_on_init_shutdown != nullptr) { + // shut down requested after we notified init complete but before we + // grabbed the lock + close_local_image(); + return false; + } + + return true; +} + +template <typename I> +void Replayer<I>::wait_for_flush() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + // ensure that we don't have two concurrent local journal replay shut downs + dout(10) << dendl; + auto ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_wait_for_flush>(this)); + m_flush_tracker.wait_for_ops(ctx); +} + +template <typename I> +void Replayer<I>::handle_wait_for_flush(int r) { + dout(10) << "r=" << r << dendl; + + shut_down_local_journal_replay(); +} + +template <typename I> +void Replayer<I>::shut_down_local_journal_replay() { + std::unique_lock locker{m_lock}; + + if (m_local_journal_replay == nullptr) { + wait_for_event_replay(); + return; + } + + // It's required to stop the local journal replay state machine prior to + // waiting for the events to complete. This is to ensure that IO is properly + // flushed (it might be batched), wait for any running ops to complete, and + // to cancel any ops waiting for their associated OnFinish events. + dout(10) << dendl; + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_shut_down_local_journal_replay>(this); + m_local_journal_replay->shut_down(true, ctx); +} + +template <typename I> +void Replayer<I>::handle_shut_down_local_journal_replay(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + if (r < 0) { + derr << "error shutting down journal replay: " << cpp_strerror(r) << dendl; + handle_replay_error(r, "failed to shut down local journal replay"); + } + + wait_for_event_replay(); +} + +template <typename I> +void Replayer<I>::wait_for_event_replay() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + dout(10) << dendl; + auto ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_wait_for_event_replay>(this)); + m_event_replay_tracker.wait_for_ops(ctx); +} + +template <typename I> +void Replayer<I>::handle_wait_for_event_replay(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + close_local_image(); +} + +template <typename I> +void Replayer<I>::close_local_image() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + if (m_state_builder->local_image_ctx == nullptr) { + stop_remote_journaler_replay(); + return; + } + + dout(10) << dendl; + if (m_local_journal_listener != nullptr) { + // blocks if listener notification is in-progress + m_local_journal->remove_listener(m_local_journal_listener); + delete m_local_journal_listener; + m_local_journal_listener = nullptr; + } + + if (m_local_journal_replay != nullptr) { + m_local_journal->stop_external_replay(); + m_local_journal_replay = nullptr; + } + + if (m_event_preprocessor != nullptr) { + image_replayer::journal::EventPreprocessor<I>::destroy( + m_event_preprocessor); + m_event_preprocessor = nullptr; + } + + m_local_journal.reset(); + + // NOTE: it's important to ensure that the local image is fully + // closed before attempting to close the remote journal in + // case the remote cluster is unreachable + ceph_assert(m_state_builder->local_image_ctx != nullptr); + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_close_local_image>(this); + auto request = image_replayer::CloseImageRequest<I>::create( + &m_state_builder->local_image_ctx, ctx); + request->send(); +} + + +template <typename I> +void Replayer<I>::handle_close_local_image(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + if (r < 0) { + derr << "error closing local iamge: " << cpp_strerror(r) << dendl; + handle_replay_error(r, "failed to close local image"); + } + + ceph_assert(m_state_builder->local_image_ctx == nullptr); + stop_remote_journaler_replay(); +} + +template <typename I> +void Replayer<I>::stop_remote_journaler_replay() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + if (m_state_builder->remote_journaler == nullptr) { + wait_for_in_flight_ops(); + return; + } else if (m_remote_replay_handler == nullptr) { + wait_for_in_flight_ops(); + return; + } + + dout(10) << dendl; + auto ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_stop_remote_journaler_replay>(this)); + m_state_builder->remote_journaler->stop_replay(ctx); +} + +template <typename I> +void Replayer<I>::handle_stop_remote_journaler_replay(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + if (r < 0) { + derr << "failed to stop remote journaler replay : " << cpp_strerror(r) + << dendl; + handle_replay_error(r, "failed to stop remote journaler replay"); + } + + delete m_remote_replay_handler; + m_remote_replay_handler = nullptr; + + wait_for_in_flight_ops(); +} + +template <typename I> +void Replayer<I>::wait_for_in_flight_ops() { + dout(10) << dendl; + if (m_remote_listener != nullptr) { + m_state_builder->remote_journaler->remove_listener(m_remote_listener); + delete m_remote_listener; + m_remote_listener = nullptr; + } + + auto ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_wait_for_in_flight_ops>(this)); + m_in_flight_op_tracker.wait_for_ops(ctx); +} + +template <typename I> +void Replayer<I>::handle_wait_for_in_flight_ops(int r) { + dout(10) << "r=" << r << dendl; + + ReplayStatusFormatter<I>::destroy(m_replay_status_formatter); + m_replay_status_formatter = nullptr; + + Context* on_init_shutdown = nullptr; + { + std::unique_lock locker{m_lock}; + ceph_assert(m_on_init_shutdown != nullptr); + std::swap(m_on_init_shutdown, on_init_shutdown); + m_state = STATE_COMPLETE; + } + on_init_shutdown->complete(m_error_code); +} + +template <typename I> +void Replayer<I>::handle_remote_journal_metadata_updated() { + dout(20) << dendl; + + std::unique_lock locker{m_lock}; + if (m_state != STATE_REPLAYING) { + return; + } + + cls::journal::Client remote_client; + int r = m_state_builder->remote_journaler->get_cached_client( + m_local_mirror_uuid, &remote_client); + if (r < 0) { + derr << "failed to retrieve client: " << cpp_strerror(r) << dendl; + return; + } + + librbd::journal::MirrorPeerClientMeta remote_client_meta; + std::string error; + r = validate_remote_client_state(remote_client, &remote_client_meta, + &m_resync_requested, &error); + if (r < 0) { + dout(0) << "client flagged disconnected, stopping image replay" << dendl; + handle_replay_complete(locker, r, error); + } +} + +template <typename I> +void Replayer<I>::schedule_flush_local_replay_task() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + std::unique_lock timer_locker{m_threads->timer_lock}; + if (m_state != STATE_REPLAYING || m_flush_local_replay_task != nullptr) { + return; + } + + dout(15) << dendl; + m_flush_local_replay_task = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_flush_local_replay_task>(this)); + m_threads->timer->add_event_after(30, m_flush_local_replay_task); +} + +template <typename I> +void Replayer<I>::cancel_flush_local_replay_task() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + std::unique_lock timer_locker{m_threads->timer_lock}; + if (m_flush_local_replay_task != nullptr) { + dout(10) << dendl; + m_threads->timer->cancel_event(m_flush_local_replay_task); + m_flush_local_replay_task = nullptr; + } +} + +template <typename I> +void Replayer<I>::handle_flush_local_replay_task(int) { + dout(15) << dendl; + + m_in_flight_op_tracker.start_op(); + auto on_finish = new LambdaContext([this](int) { + std::unique_lock locker{m_lock}; + + { + std::unique_lock timer_locker{m_threads->timer_lock}; + m_flush_local_replay_task = nullptr; + } + + notify_status_updated(); + m_in_flight_op_tracker.finish_op(); + }); + flush_local_replay(on_finish); +} + +template <typename I> +void Replayer<I>::flush_local_replay(Context* on_flush) { + std::unique_lock locker{m_lock}; + if (m_state != STATE_REPLAYING) { + locker.unlock(); + on_flush->complete(0); + return; + } else if (m_local_journal_replay == nullptr) { + // raced w/ a tag creation stop/start, which implies that + // the replay is flushed + locker.unlock(); + flush_commit_position(on_flush); + return; + } + + dout(15) << dendl; + auto ctx = new LambdaContext( + [this, on_flush](int r) { + handle_flush_local_replay(on_flush, r); + }); + m_local_journal_replay->flush(ctx); +} + +template <typename I> +void Replayer<I>::handle_flush_local_replay(Context* on_flush, int r) { + dout(15) << "r=" << r << dendl; + if (r < 0) { + derr << "error flushing local replay: " << cpp_strerror(r) << dendl; + on_flush->complete(r); + return; + } + + flush_commit_position(on_flush); +} + +template <typename I> +void Replayer<I>::flush_commit_position(Context* on_flush) { + std::unique_lock locker{m_lock}; + if (m_state != STATE_REPLAYING) { + locker.unlock(); + on_flush->complete(0); + return; + } + + dout(15) << dendl; + auto ctx = new LambdaContext( + [this, on_flush](int r) { + handle_flush_commit_position(on_flush, r); + }); + m_state_builder->remote_journaler->flush_commit_position(ctx); +} + +template <typename I> +void Replayer<I>::handle_flush_commit_position(Context* on_flush, int r) { + dout(15) << "r=" << r << dendl; + if (r < 0) { + derr << "error flushing remote journal commit position: " + << cpp_strerror(r) << dendl; + } + + on_flush->complete(r); +} + +template <typename I> +void Replayer<I>::handle_replay_error(int r, const std::string &error) { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + if (m_error_code == 0) { + m_error_code = r; + m_error_description = error; + } +} + +template <typename I> +bool Replayer<I>::is_replay_complete() const { + std::unique_lock locker{m_lock}; + return is_replay_complete(locker); +} + +template <typename I> +bool Replayer<I>::is_replay_complete( + const std::unique_lock<ceph::mutex>&) const { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + return (m_state == STATE_COMPLETE); +} + +template <typename I> +void Replayer<I>::handle_replay_complete(int r, const std::string &error) { + std::unique_lock locker{m_lock}; + handle_replay_complete(locker, r, error); +} + +template <typename I> +void Replayer<I>::handle_replay_complete( + const std::unique_lock<ceph::mutex>&, int r, const std::string &error) { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + dout(10) << "r=" << r << ", error=" << error << dendl; + if (r < 0) { + derr << "replay encountered an error: " << cpp_strerror(r) << dendl; + handle_replay_error(r, error); + } + + if (m_state != STATE_REPLAYING) { + return; + } + + m_state = STATE_COMPLETE; + notify_status_updated(); +} + +template <typename I> +void Replayer<I>::handle_replay_ready() { + std::unique_lock locker{m_lock}; + handle_replay_ready(locker); +} + +template <typename I> +void Replayer<I>::handle_replay_ready( + std::unique_lock<ceph::mutex>& locker) { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + dout(20) << dendl; + if (is_replay_complete(locker)) { + return; + } + + if (!m_state_builder->remote_journaler->try_pop_front(&m_replay_entry, + &m_replay_tag_tid)) { + dout(20) << "no entries ready for replay" << dendl; + return; + } + + // can safely drop lock once the entry is tracked + m_event_replay_tracker.start_op(); + locker.unlock(); + + dout(20) << "entry tid=" << m_replay_entry.get_commit_tid() + << "tag_tid=" << m_replay_tag_tid << dendl; + if (!m_replay_tag_valid || m_replay_tag.tid != m_replay_tag_tid) { + // must allocate a new local journal tag prior to processing + replay_flush(); + return; + } + + preprocess_entry(); +} + +template <typename I> +void Replayer<I>::replay_flush() { + dout(10) << dendl; + m_flush_tracker.start_op(); + + // shut down the replay to flush all IO and ops and create a new + // replayer to handle the new tag epoch + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_replay_flush_shut_down>(this); + ceph_assert(m_local_journal_replay != nullptr); + m_local_journal_replay->shut_down(false, ctx); +} + +template <typename I> +void Replayer<I>::handle_replay_flush_shut_down(int r) { + std::unique_lock locker{m_lock}; + dout(10) << "r=" << r << dendl; + + ceph_assert(m_local_journal != nullptr); + ceph_assert(m_local_journal_listener != nullptr); + + // blocks if listener notification is in-progress + m_local_journal->remove_listener(m_local_journal_listener); + delete m_local_journal_listener; + m_local_journal_listener = nullptr; + + m_local_journal->stop_external_replay(); + m_local_journal_replay = nullptr; + m_local_journal.reset(); + + if (r < 0) { + locker.unlock(); + + handle_replay_flush(r); + return; + } + + // journal might have been closed now that we stopped external replay + auto local_image_ctx = m_state_builder->local_image_ctx; + std::shared_lock local_image_locker{local_image_ctx->image_lock}; + m_local_journal = local_image_ctx->journal; + if (m_local_journal == nullptr) { + local_image_locker.unlock(); + locker.unlock(); + + derr << "local image journal closed" << dendl; + handle_replay_flush(-EINVAL); + return; + } + + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_replay_flush>(this); + m_local_journal->start_external_replay(&m_local_journal_replay, ctx); +} + +template <typename I> +void Replayer<I>::handle_replay_flush(int r) { + std::unique_lock locker{m_lock}; + dout(10) << "r=" << r << dendl; + m_flush_tracker.finish_op(); + + if (r < 0) { + derr << "replay flush encountered an error: " << cpp_strerror(r) << dendl; + handle_replay_complete(locker, r, "replay flush encountered an error"); + m_event_replay_tracker.finish_op(); + return; + } else if (is_replay_complete(locker)) { + m_event_replay_tracker.finish_op(); + return; + } + + // check for resync/promotion state after adding listener + if (!add_local_journal_listener(locker)) { + m_event_replay_tracker.finish_op(); + return; + } + locker.unlock(); + + get_remote_tag(); +} + +template <typename I> +void Replayer<I>::get_remote_tag() { + dout(15) << "tag_tid: " << m_replay_tag_tid << dendl; + + Context *ctx = create_context_callback< + Replayer, &Replayer<I>::handle_get_remote_tag>(this); + m_state_builder->remote_journaler->get_tag(m_replay_tag_tid, &m_replay_tag, + ctx); +} + +template <typename I> +void Replayer<I>::handle_get_remote_tag(int r) { + dout(15) << "r=" << r << dendl; + + if (r == 0) { + try { + auto it = m_replay_tag.data.cbegin(); + decode(m_replay_tag_data, it); + } catch (const buffer::error &err) { + r = -EBADMSG; + } + } + + if (r < 0) { + derr << "failed to retrieve remote tag " << m_replay_tag_tid << ": " + << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to retrieve remote tag"); + m_event_replay_tracker.finish_op(); + return; + } + + m_replay_tag_valid = true; + dout(15) << "decoded remote tag " << m_replay_tag_tid << ": " + << m_replay_tag_data << dendl; + + allocate_local_tag(); +} + +template <typename I> +void Replayer<I>::allocate_local_tag() { + dout(15) << dendl; + + std::string mirror_uuid = m_replay_tag_data.mirror_uuid; + if (mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID) { + mirror_uuid = m_state_builder->remote_mirror_uuid; + } else if (mirror_uuid == m_local_mirror_uuid) { + mirror_uuid = librbd::Journal<>::LOCAL_MIRROR_UUID; + } else if (mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID) { + // handle possible edge condition where daemon can failover and + // the local image has already been promoted/demoted + auto local_tag_data = m_local_journal->get_tag_data(); + if (local_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID && + (local_tag_data.predecessor.commit_valid && + local_tag_data.predecessor.mirror_uuid == + librbd::Journal<>::LOCAL_MIRROR_UUID)) { + dout(15) << "skipping stale demotion event" << dendl; + handle_process_entry_safe(m_replay_entry, m_replay_bytes, + m_replay_start_time, 0); + handle_replay_ready(); + return; + } else { + dout(5) << "encountered image demotion: stopping" << dendl; + handle_replay_complete(0, ""); + } + } + + librbd::journal::TagPredecessor predecessor(m_replay_tag_data.predecessor); + if (predecessor.mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID) { + predecessor.mirror_uuid = m_state_builder->remote_mirror_uuid; + } else if (predecessor.mirror_uuid == m_local_mirror_uuid) { + predecessor.mirror_uuid = librbd::Journal<>::LOCAL_MIRROR_UUID; + } + + dout(15) << "mirror_uuid=" << mirror_uuid << ", " + << "predecessor=" << predecessor << ", " + << "replay_tag_tid=" << m_replay_tag_tid << dendl; + Context *ctx = create_context_callback< + Replayer, &Replayer<I>::handle_allocate_local_tag>(this); + m_local_journal->allocate_tag(mirror_uuid, predecessor, ctx); +} + +template <typename I> +void Replayer<I>::handle_allocate_local_tag(int r) { + dout(15) << "r=" << r << ", " + << "tag_tid=" << m_local_journal->get_tag_tid() << dendl; + if (r < 0) { + derr << "failed to allocate journal tag: " << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to allocate journal tag"); + m_event_replay_tracker.finish_op(); + return; + } + + preprocess_entry(); +} + +template <typename I> +void Replayer<I>::preprocess_entry() { + dout(20) << "preprocessing entry tid=" << m_replay_entry.get_commit_tid() + << dendl; + + bufferlist data = m_replay_entry.get_data(); + auto it = data.cbegin(); + int r = m_local_journal_replay->decode(&it, &m_event_entry); + if (r < 0) { + derr << "failed to decode journal event" << dendl; + handle_replay_complete(r, "failed to decode journal event"); + m_event_replay_tracker.finish_op(); + return; + } + + m_replay_bytes = data.length(); + uint32_t delay = calculate_replay_delay( + m_event_entry.timestamp, + m_state_builder->local_image_ctx->mirroring_replay_delay); + if (delay == 0) { + handle_preprocess_entry_ready(0); + return; + } + + std::unique_lock locker{m_lock}; + if (is_replay_complete(locker)) { + // don't schedule a delayed replay task if a shut-down is in-progress + m_event_replay_tracker.finish_op(); + return; + } + + dout(20) << "delaying replay by " << delay << " sec" << dendl; + std::unique_lock timer_locker{m_threads->timer_lock}; + ceph_assert(m_delayed_preprocess_task == nullptr); + m_delayed_preprocess_task = create_context_callback< + Replayer<I>, &Replayer<I>::handle_delayed_preprocess_task>(this); + m_threads->timer->add_event_after(delay, m_delayed_preprocess_task); +} + +template <typename I> +void Replayer<I>::handle_delayed_preprocess_task(int r) { + dout(20) << "r=" << r << dendl; + + ceph_assert(ceph_mutex_is_locked_by_me(m_threads->timer_lock)); + m_delayed_preprocess_task = nullptr; + + m_threads->work_queue->queue(create_context_callback< + Replayer, &Replayer<I>::handle_preprocess_entry_ready>(this), 0); +} + +template <typename I> +void Replayer<I>::handle_preprocess_entry_ready(int r) { + dout(20) << "r=" << r << dendl; + ceph_assert(r == 0); + + m_replay_start_time = ceph_clock_now(); + if (!m_event_preprocessor->is_required(m_event_entry)) { + process_entry(); + return; + } + + Context *ctx = create_context_callback< + Replayer, &Replayer<I>::handle_preprocess_entry_safe>(this); + m_event_preprocessor->preprocess(&m_event_entry, ctx); +} + +template <typename I> +void Replayer<I>::handle_preprocess_entry_safe(int r) { + dout(20) << "r=" << r << dendl; + + if (r < 0) { + if (r == -ECANCELED) { + handle_replay_complete(0, "lost exclusive lock"); + } else { + derr << "failed to preprocess journal event" << dendl; + handle_replay_complete(r, "failed to preprocess journal event"); + } + + m_event_replay_tracker.finish_op(); + return; + } + + process_entry(); +} + +template <typename I> +void Replayer<I>::process_entry() { + dout(20) << "processing entry tid=" << m_replay_entry.get_commit_tid() + << dendl; + + Context *on_ready = create_context_callback< + Replayer, &Replayer<I>::handle_process_entry_ready>(this); + Context *on_commit = new C_ReplayCommitted(this, std::move(m_replay_entry), + m_replay_bytes, + m_replay_start_time); + + m_local_journal_replay->process(m_event_entry, on_ready, on_commit); +} + +template <typename I> +void Replayer<I>::handle_process_entry_ready(int r) { + std::unique_lock locker{m_lock}; + + dout(20) << dendl; + ceph_assert(r == 0); + + bool update_status = false; + { + auto local_image_ctx = m_state_builder->local_image_ctx; + std::shared_lock image_locker{local_image_ctx->image_lock}; + auto image_spec = util::compute_image_spec(local_image_ctx->md_ctx, + local_image_ctx->name); + if (m_image_spec != image_spec) { + m_image_spec = image_spec; + update_status = true; + } + } + + m_replay_status_formatter->handle_entry_processed(m_replay_bytes); + + if (update_status) { + unregister_perf_counters(); + register_perf_counters(); + notify_status_updated(); + } + + // attempt to process the next event + handle_replay_ready(locker); +} + +template <typename I> +void Replayer<I>::handle_process_entry_safe( + const ReplayEntry &replay_entry, uint64_t replay_bytes, + const utime_t &replay_start_time, int r) { + dout(20) << "commit_tid=" << replay_entry.get_commit_tid() << ", r=" << r + << dendl; + + if (r < 0) { + derr << "failed to commit journal event: " << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to commit journal event"); + } else { + ceph_assert(m_state_builder->remote_journaler != nullptr); + m_state_builder->remote_journaler->committed(replay_entry); + } + + auto latency = ceph_clock_now() - replay_start_time; + if (g_journal_perf_counters) { + g_journal_perf_counters->inc(l_rbd_mirror_replay); + g_journal_perf_counters->inc(l_rbd_mirror_replay_bytes, replay_bytes); + g_journal_perf_counters->tinc(l_rbd_mirror_replay_latency, latency); + } + + auto ctx = new LambdaContext( + [this, replay_bytes, latency](int r) { + std::unique_lock locker{m_lock}; + schedule_flush_local_replay_task(); + + if (m_perf_counters) { + m_perf_counters->inc(l_rbd_mirror_replay); + m_perf_counters->inc(l_rbd_mirror_replay_bytes, replay_bytes); + m_perf_counters->tinc(l_rbd_mirror_replay_latency, latency); + } + + m_event_replay_tracker.finish_op(); + }); + m_threads->work_queue->queue(ctx, 0); +} + +template <typename I> +void Replayer<I>::handle_resync_image() { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + m_resync_requested = true; + handle_replay_complete(locker, 0, "resync requested"); +} + +template <typename I> +void Replayer<I>::notify_status_updated() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + dout(10) << dendl; + + auto ctx = new C_TrackedOp(m_in_flight_op_tracker, new LambdaContext( + [this](int) { + m_replayer_listener->handle_notification(); + })); + m_threads->work_queue->queue(ctx, 0); +} + +template <typename I> +void Replayer<I>::cancel_delayed_preprocess_task() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + bool canceled_delayed_preprocess_task = false; + { + std::unique_lock timer_locker{m_threads->timer_lock}; + if (m_delayed_preprocess_task != nullptr) { + dout(10) << dendl; + canceled_delayed_preprocess_task = m_threads->timer->cancel_event( + m_delayed_preprocess_task); + ceph_assert(canceled_delayed_preprocess_task); + m_delayed_preprocess_task = nullptr; + } + } + + if (canceled_delayed_preprocess_task) { + // wake up sleeping replay + m_event_replay_tracker.finish_op(); + } +} + +template <typename I> +int Replayer<I>::validate_remote_client_state( + const cls::journal::Client& remote_client, + librbd::journal::MirrorPeerClientMeta* remote_client_meta, + bool* resync_requested, std::string* error) { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + if (!util::decode_client_meta(remote_client, remote_client_meta)) { + // require operator intervention since the data is corrupt + *error = "error retrieving remote journal client"; + return -EBADMSG; + } + + auto local_image_ctx = m_state_builder->local_image_ctx; + dout(5) << "image_id=" << local_image_ctx->id << ", " + << "remote_client_meta.image_id=" + << remote_client_meta->image_id << ", " + << "remote_client.state=" << remote_client.state << dendl; + if (remote_client_meta->image_id == local_image_ctx->id && + remote_client.state != cls::journal::CLIENT_STATE_CONNECTED) { + dout(5) << "client flagged disconnected, stopping image replay" << dendl; + if (local_image_ctx->config.template get_val<bool>( + "rbd_mirroring_resync_after_disconnect")) { + dout(10) << "disconnected: automatic resync" << dendl; + *resync_requested = true; + *error = "disconnected: automatic resync"; + return -ENOTCONN; + } else { + dout(10) << "disconnected" << dendl; + *error = "disconnected"; + return -ENOTCONN; + } + } + + return 0; +} + +template <typename I> +void Replayer<I>::register_perf_counters() { + dout(5) << dendl; + + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + ceph_assert(m_perf_counters == nullptr); + + auto cct = static_cast<CephContext *>(m_state_builder->local_image_ctx->cct); + auto prio = cct->_conf.get_val<int64_t>("rbd_mirror_image_perf_stats_prio"); + PerfCountersBuilder plb(g_ceph_context, "rbd_mirror_image_" + m_image_spec, + l_rbd_mirror_journal_first, l_rbd_mirror_journal_last); + plb.add_u64_counter(l_rbd_mirror_replay, "replay", "Replays", "r", prio); + plb.add_u64_counter(l_rbd_mirror_replay_bytes, "replay_bytes", + "Replayed data", "rb", prio, unit_t(UNIT_BYTES)); + plb.add_time_avg(l_rbd_mirror_replay_latency, "replay_latency", + "Replay latency", "rl", prio); + m_perf_counters = plb.create_perf_counters(); + g_ceph_context->get_perfcounters_collection()->add(m_perf_counters); +} + +template <typename I> +void Replayer<I>::unregister_perf_counters() { + dout(5) << dendl; + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + PerfCounters *perf_counters = nullptr; + std::swap(perf_counters, m_perf_counters); + + if (perf_counters != nullptr) { + g_ceph_context->get_perfcounters_collection()->remove(perf_counters); + delete perf_counters; + } +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::Replayer<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/journal/Replayer.h b/src/tools/rbd_mirror/image_replayer/journal/Replayer.h new file mode 100644 index 000000000..6b1f36d9c --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/Replayer.h @@ -0,0 +1,323 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_REPLAYER_H +#define RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_REPLAYER_H + +#include "tools/rbd_mirror/image_replayer/Replayer.h" +#include "include/utime.h" +#include "common/AsyncOpTracker.h" +#include "common/ceph_mutex.h" +#include "common/RefCountedObj.h" +#include "cls/journal/cls_journal_types.h" +#include "journal/ReplayEntry.h" +#include "librbd/ImageCtx.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include <string> +#include <type_traits> + +namespace journal { class Journaler; } +namespace librbd { + +struct ImageCtx; +namespace journal { template <typename I> class Replay; } + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <typename> struct Threads; + +namespace image_replayer { + +struct ReplayerListener; + +namespace journal { + +template <typename> class EventPreprocessor; +template <typename> class ReplayStatusFormatter; +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class Replayer : public image_replayer::Replayer { +public: + typedef typename librbd::journal::TypeTraits<ImageCtxT>::Journaler Journaler; + + static Replayer* create( + Threads<ImageCtxT>* threads, + const std::string& local_mirror_uuid, + StateBuilder<ImageCtxT>* state_builder, + ReplayerListener* replayer_listener) { + return new Replayer(threads, local_mirror_uuid, state_builder, + replayer_listener); + } + + Replayer( + Threads<ImageCtxT>* threads, + const std::string& local_mirror_uuid, + StateBuilder<ImageCtxT>* state_builder, + ReplayerListener* replayer_listener); + ~Replayer(); + + void destroy() override { + delete this; + } + + void init(Context* on_finish) override; + void shut_down(Context* on_finish) override; + + void flush(Context* on_finish) override; + + bool get_replay_status(std::string* description, Context* on_finish) override; + + bool is_replaying() const override { + std::unique_lock locker{m_lock}; + return (m_state == STATE_REPLAYING); + } + + bool is_resync_requested() const override { + std::unique_lock locker(m_lock); + return m_resync_requested; + } + + int get_error_code() const override { + std::unique_lock locker(m_lock); + return m_error_code; + } + + std::string get_error_description() const override { + std::unique_lock locker(m_lock); + return m_error_description; + } + + std::string get_image_spec() const { + std::unique_lock locker(m_lock); + return m_image_spec; + } + +private: + /** + * @verbatim + * + * <init> + * | + * v (error) + * INIT_REMOTE_JOURNALER * * * * * * * * * * * * * * * * * * * + * | * + * v (error) * + * START_EXTERNAL_REPLAY * * * * * * * * * * * * * * * * * * * + * | * + * | /--------------------------------------------\ * + * | | | * + * v v (asok flush) | * + * REPLAYING -------------> LOCAL_REPLAY_FLUSH | * + * | \ | | * + * | | v | * + * | | FLUSH_COMMIT_POSITION | * + * | | | | * + * | | \--------------------/| * + * | | | * + * | | (entries available) | * + * | \-----------> REPLAY_READY | * + * | | | * + * | | (skip if not | * + * | v needed) (error) * + * | REPLAY_FLUSH * * * * * * * * * * + * | | | * * + * | | (skip if not | * * + * | v needed) (error) * * + * | GET_REMOTE_TAG * * * * * * * * * + * | | | * * + * | | (skip if not | * * + * | v needed) (error) * * + * | ALLOCATE_LOCAL_TAG * * * * * * * + * | | | * * + * | v (error) * * + * | PREPROCESS_ENTRY * * * * * * * * + * | | | * * + * | v (error) * * + * | PROCESS_ENTRY * * * * * * * * * * + * | | | * * + * | \---------------------/ * * + * v (shutdown) * * + * REPLAY_COMPLETE < * * * * * * * * * * * * * * * * * * * * + * | * + * v * + * WAIT_FOR_FLUSH * + * | * + * v * + * SHUT_DOWN_LOCAL_JOURNAL_REPLAY * + * | * + * v * + * WAIT_FOR_REPLAY * + * | * + * v * + * CLOSE_LOCAL_IMAGE < * * * * * * * * * * * * * * * * * * * * + * | + * v (skip if not started) + * STOP_REMOTE_JOURNALER_REPLAY + * | + * v + * WAIT_FOR_IN_FLIGHT_OPS + * | + * v + * <shutdown> + * + * @endverbatim + */ + + typedef typename librbd::journal::TypeTraits<ImageCtxT>::ReplayEntry ReplayEntry; + + enum State { + STATE_INIT, + STATE_REPLAYING, + STATE_COMPLETE + }; + + struct C_ReplayCommitted; + struct RemoteJournalerListener; + struct RemoteReplayHandler; + struct LocalJournalListener; + + Threads<ImageCtxT>* m_threads; + std::string m_local_mirror_uuid; + StateBuilder<ImageCtxT>* m_state_builder; + ReplayerListener* m_replayer_listener; + + mutable ceph::mutex m_lock; + + std::string m_image_spec; + Context* m_on_init_shutdown = nullptr; + + State m_state = STATE_INIT; + int m_error_code = 0; + std::string m_error_description; + bool m_resync_requested = false; + + ceph::ref_t<typename std::remove_pointer<decltype(ImageCtxT::journal)>::type> + m_local_journal; + RemoteJournalerListener* m_remote_listener = nullptr; + + librbd::journal::Replay<ImageCtxT>* m_local_journal_replay = nullptr; + EventPreprocessor<ImageCtxT>* m_event_preprocessor = nullptr; + ReplayStatusFormatter<ImageCtxT>* m_replay_status_formatter = nullptr; + RemoteReplayHandler* m_remote_replay_handler = nullptr; + LocalJournalListener* m_local_journal_listener = nullptr; + + PerfCounters *m_perf_counters = nullptr; + + ReplayEntry m_replay_entry; + uint64_t m_replay_bytes = 0; + utime_t m_replay_start_time; + bool m_replay_tag_valid = false; + uint64_t m_replay_tag_tid = 0; + cls::journal::Tag m_replay_tag; + librbd::journal::TagData m_replay_tag_data; + librbd::journal::EventEntry m_event_entry; + + AsyncOpTracker m_flush_tracker; + + AsyncOpTracker m_event_replay_tracker; + Context *m_delayed_preprocess_task = nullptr; + + AsyncOpTracker m_in_flight_op_tracker; + Context *m_flush_local_replay_task = nullptr; + + void handle_remote_journal_metadata_updated(); + + void schedule_flush_local_replay_task(); + void cancel_flush_local_replay_task(); + void handle_flush_local_replay_task(int r); + + void flush_local_replay(Context* on_flush); + void handle_flush_local_replay(Context* on_flush, int r); + + void flush_commit_position(Context* on_flush); + void handle_flush_commit_position(Context* on_flush, int r); + + void init_remote_journaler(); + void handle_init_remote_journaler(int r); + + void start_external_replay(std::unique_lock<ceph::mutex>& locker); + void handle_start_external_replay(int r); + + bool add_local_journal_listener(std::unique_lock<ceph::mutex>& locker); + + bool notify_init_complete(std::unique_lock<ceph::mutex>& locker); + + void wait_for_flush(); + void handle_wait_for_flush(int r); + + void shut_down_local_journal_replay(); + void handle_shut_down_local_journal_replay(int r); + + void wait_for_event_replay(); + void handle_wait_for_event_replay(int r); + + void close_local_image(); + void handle_close_local_image(int r); + + void stop_remote_journaler_replay(); + void handle_stop_remote_journaler_replay(int r); + + void wait_for_in_flight_ops(); + void handle_wait_for_in_flight_ops(int r); + + void replay_flush(); + void handle_replay_flush_shut_down(int r); + void handle_replay_flush(int r); + + void get_remote_tag(); + void handle_get_remote_tag(int r); + + void allocate_local_tag(); + void handle_allocate_local_tag(int r); + + void handle_replay_error(int r, const std::string &error); + + bool is_replay_complete() const; + bool is_replay_complete(const std::unique_lock<ceph::mutex>& locker) const; + + void handle_replay_complete(int r, const std::string &error_desc); + void handle_replay_complete(const std::unique_lock<ceph::mutex>&, + int r, const std::string &error_desc); + void handle_replay_ready(); + void handle_replay_ready(std::unique_lock<ceph::mutex>& locker); + + void preprocess_entry(); + void handle_delayed_preprocess_task(int r); + void handle_preprocess_entry_ready(int r); + void handle_preprocess_entry_safe(int r); + + void process_entry(); + void handle_process_entry_ready(int r); + void handle_process_entry_safe(const ReplayEntry& replay_entry, + uint64_t relay_bytes, + const utime_t &replay_start_time, int r); + + void handle_resync_image(); + + void notify_status_updated(); + + void cancel_delayed_preprocess_task(); + + int validate_remote_client_state( + const cls::journal::Client& remote_client, + librbd::journal::MirrorPeerClientMeta* remote_client_meta, + bool* resync_requested, std::string* error); + + void register_perf_counters(); + void unregister_perf_counters(); + +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::Replayer<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_REPLAYER_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc new file mode 100644 index 000000000..5f1fb0e2f --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc @@ -0,0 +1,149 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "StateBuilder.h" +#include "include/ceph_assert.h" +#include "include/Context.h" +#include "common/debug.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "tools/rbd_mirror/image_replayer/journal/CreateLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/journal/PrepareReplayRequest.h" +#include "tools/rbd_mirror/image_replayer/journal/Replayer.h" +#include "tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "StateBuilder: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template <typename I> +StateBuilder<I>::StateBuilder(const std::string& global_image_id) + : image_replayer::StateBuilder<I>(global_image_id) { +} + +template <typename I> +StateBuilder<I>::~StateBuilder() { + ceph_assert(remote_journaler == nullptr); +} + +template <typename I> +void StateBuilder<I>::close(Context* on_finish) { + dout(10) << dendl; + + // close the remote journaler after closing the local image + // in case we have lost contact w/ the remote cluster and + // will block + on_finish = new LambdaContext([this, on_finish](int) { + shut_down_remote_journaler(on_finish); + }); + on_finish = new LambdaContext([this, on_finish](int) { + this->close_local_image(on_finish); + }); + this->close_remote_image(on_finish); +} + +template <typename I> +bool StateBuilder<I>::is_disconnected() const { + return (remote_client_state == cls::journal::CLIENT_STATE_DISCONNECTED); +} + +template <typename I> +bool StateBuilder<I>::is_linked_impl() const { + ceph_assert(!this->remote_mirror_uuid.empty()); + return (local_primary_mirror_uuid == this->remote_mirror_uuid); +} + +template <typename I> +cls::rbd::MirrorImageMode StateBuilder<I>::get_mirror_image_mode() const { + return cls::rbd::MIRROR_IMAGE_MODE_JOURNAL; +} + +template <typename I> +image_sync::SyncPointHandler* StateBuilder<I>::create_sync_point_handler() { + dout(10) << dendl; + + this->m_sync_point_handler = SyncPointHandler<I>::create(this); + return this->m_sync_point_handler; +} + +template <typename I> +BaseRequest* StateBuilder<I>::create_local_image_request( + Threads<I>* threads, + librados::IoCtx& local_io_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + Context* on_finish) { + return CreateLocalImageRequest<I>::create( + threads, local_io_ctx, this->remote_image_ctx, this->global_image_id, + pool_meta_cache, progress_ctx, this, on_finish); +} + +template <typename I> +BaseRequest* StateBuilder<I>::create_prepare_replay_request( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + bool* resync_requested, + bool* syncing, + Context* on_finish) { + return PrepareReplayRequest<I>::create( + local_mirror_uuid, progress_ctx, this, resync_requested, syncing, + on_finish); +} + +template <typename I> +image_replayer::Replayer* StateBuilder<I>::create_replayer( + Threads<I>* threads, + InstanceWatcher<I>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + ReplayerListener* replayer_listener) { + return Replayer<I>::create( + threads, local_mirror_uuid, this, replayer_listener); +} + +template <typename I> +void StateBuilder<I>::shut_down_remote_journaler(Context* on_finish) { + if (remote_journaler == nullptr) { + on_finish->complete(0); + return; + } + + dout(10) << dendl; + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_shut_down_remote_journaler(r, on_finish); + }); + remote_journaler->shut_down(ctx); +} + +template <typename I> +void StateBuilder<I>::handle_shut_down_remote_journaler(int r, + Context* on_finish) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to shut down remote journaler: " << cpp_strerror(r) + << dendl; + } + + delete remote_journaler; + remote_journaler = nullptr; + on_finish->complete(r); +} + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::StateBuilder<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h new file mode 100644 index 000000000..790d1390b --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h @@ -0,0 +1,94 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_STATE_BUILDER_H +#define CEPH_RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_STATE_BUILDER_H + +#include "tools/rbd_mirror/image_replayer/StateBuilder.h" +#include "cls/journal/cls_journal_types.h" +#include "librbd/journal/Types.h" +#include "librbd/journal/TypeTraits.h" +#include <string> + +struct Context; + +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template <typename> class SyncPointHandler; + +template <typename ImageCtxT> +class StateBuilder : public image_replayer::StateBuilder<ImageCtxT> { +public: + typedef librbd::journal::TypeTraits<ImageCtxT> TypeTraits; + typedef typename TypeTraits::Journaler Journaler; + + static StateBuilder* create(const std::string& global_image_id) { + return new StateBuilder(global_image_id); + } + + StateBuilder(const std::string& global_image_id); + ~StateBuilder() override; + + void close(Context* on_finish) override; + + bool is_disconnected() const override; + + cls::rbd::MirrorImageMode get_mirror_image_mode() const override; + + image_sync::SyncPointHandler* create_sync_point_handler() override; + + bool replay_requires_remote_image() const override { + return false; + } + + BaseRequest* create_local_image_request( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + Context* on_finish) override; + + BaseRequest* create_prepare_replay_request( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + bool* resync_requested, + bool* syncing, + Context* on_finish) override; + + image_replayer::Replayer* create_replayer( + Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + ReplayerListener* replayer_listener) override; + + std::string local_primary_mirror_uuid; + + Journaler* remote_journaler = nullptr; + cls::journal::ClientState remote_client_state = + cls::journal::CLIENT_STATE_CONNECTED; + librbd::journal::MirrorPeerClientMeta remote_client_meta; + + SyncPointHandler<ImageCtxT>* sync_point_handler = nullptr; + +private: + bool is_linked_impl() const override; + + void shut_down_remote_journaler(Context* on_finish); + void handle_shut_down_remote_journaler(int r, Context* on_finish); +}; + +} // namespace journal +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::StateBuilder<librbd::ImageCtx>; + +#endif // CEPH_RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_STATE_BUILDER_H diff --git a/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.cc b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.cc new file mode 100644 index 000000000..66d13e555 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.cc @@ -0,0 +1,109 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "SyncPointHandler.h" +#include "StateBuilder.h" +#include "include/ceph_assert.h" +#include "include/Context.h" +#include "common/debug.h" +#include "common/errno.h" +#include "journal/Journaler.h" +#include "librbd/ImageCtx.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::journal::" \ + << "SyncPointHandler: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template <typename I> +SyncPointHandler<I>::SyncPointHandler(StateBuilder<I>* state_builder) + : m_state_builder(state_builder), + m_client_meta_copy(state_builder->remote_client_meta) { +} + +template <typename I> +typename SyncPointHandler<I>::SyncPoints +SyncPointHandler<I>::get_sync_points() const { + SyncPoints sync_points; + for (auto& sync_point : m_client_meta_copy.sync_points) { + sync_points.emplace_back( + sync_point.snap_namespace, + sync_point.snap_name, + sync_point.from_snap_name, + sync_point.object_number); + } + return sync_points; +} + +template <typename I> +librbd::SnapSeqs SyncPointHandler<I>::get_snap_seqs() const { + return m_client_meta_copy.snap_seqs; +} + +template <typename I> +void SyncPointHandler<I>::update_sync_points( + const librbd::SnapSeqs& snap_seqs, const SyncPoints& sync_points, + bool sync_complete, Context* on_finish) { + dout(10) << dendl; + + if (sync_complete && sync_points.empty()) { + m_client_meta_copy.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING; + } + + m_client_meta_copy.snap_seqs = snap_seqs; + m_client_meta_copy.sync_points.clear(); + for (auto& sync_point : sync_points) { + m_client_meta_copy.sync_points.emplace_back( + sync_point.snap_namespace, + sync_point.snap_name, + sync_point.from_snap_name, + sync_point.object_number); + + if (sync_point.object_number) { + m_client_meta_copy.sync_object_count = std::max( + m_client_meta_copy.sync_object_count, *sync_point.object_number + 1); + } + } + + dout(20) << "client_meta=" << m_client_meta_copy << dendl; + bufferlist client_data_bl; + librbd::journal::ClientData client_data{m_client_meta_copy}; + encode(client_data, client_data_bl); + + auto ctx = new LambdaContext([this, on_finish](int r) { + handle_update_sync_points(r, on_finish); + }); + m_state_builder->remote_journaler->update_client(client_data_bl, ctx); +} + +template <typename I> +void SyncPointHandler<I>::handle_update_sync_points(int r, Context* on_finish) { + dout(10) << "r=" << r << dendl; + + if (r >= 0) { + m_state_builder->remote_client_meta.snap_seqs = + m_client_meta_copy.snap_seqs; + m_state_builder->remote_client_meta.sync_points = + m_client_meta_copy.sync_points; + } else { + derr << "failed to update remote journal client meta for image " + << m_state_builder->global_image_id << ": " << cpp_strerror(r) + << dendl; + } + + on_finish->complete(r); +} + +} // namespace journal +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::journal::SyncPointHandler<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h new file mode 100644 index 000000000..b4f492c19 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/journal/SyncPointHandler.h @@ -0,0 +1,55 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_SYNC_POINT_HANDLER_H +#define RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_SYNC_POINT_HANDLER_H + +#include "tools/rbd_mirror/image_sync/Types.h" +#include "librbd/journal/Types.h" + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace journal { + +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class SyncPointHandler : public image_sync::SyncPointHandler { +public: + using SyncPoint = image_sync::SyncPoint; + using SyncPoints = image_sync::SyncPoints; + + static SyncPointHandler* create(StateBuilder<ImageCtxT>* state_builder) { + return new SyncPointHandler(state_builder); + } + SyncPointHandler(StateBuilder<ImageCtxT>* state_builder); + + SyncPoints get_sync_points() const override; + librbd::SnapSeqs get_snap_seqs() const override; + + void update_sync_points(const librbd::SnapSeqs& snap_seqs, + const SyncPoints& sync_points, + bool sync_complete, + Context* on_finish) override; + +private: + StateBuilder<ImageCtxT>* m_state_builder; + + librbd::journal::MirrorPeerClientMeta m_client_meta_copy; + + void handle_update_sync_points(int r, Context* on_finish); + +}; + +} // namespace journal +} // namespace image_sync +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::journal::SyncPointHandler<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_SYNC_POINT_HANDLER_H diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.cc b/src/tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.cc new file mode 100644 index 000000000..2ed321738 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.cc @@ -0,0 +1,658 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "ApplyImageStateRequest.h" +#include "common/debug.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/image/GetMetadataRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/Utils.h" +#include <boost/algorithm/string/predicate.hpp> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::snapshot::" \ + << "ApplyImageStateRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +ApplyImageStateRequest<I>::ApplyImageStateRequest( + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + I* local_image_ctx, + I* remote_image_ctx, + librbd::mirror::snapshot::ImageState image_state, + Context* on_finish) + : m_local_mirror_uuid(local_mirror_uuid), + m_remote_mirror_uuid(remote_mirror_uuid), + m_local_image_ctx(local_image_ctx), + m_remote_image_ctx(remote_image_ctx), + m_image_state(image_state), + m_on_finish(on_finish) { + dout(15) << "image_state=" << m_image_state << dendl; + + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + m_features = m_local_image_ctx->features & ~RBD_FEATURES_IMPLICIT_ENABLE; + compute_local_to_remote_snap_ids(); +} + +template <typename I> +void ApplyImageStateRequest<I>::send() { + rename_image(); +} + +template <typename I> +void ApplyImageStateRequest<I>::rename_image() { + std::shared_lock owner_locker{m_local_image_ctx->owner_lock}; + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + if (m_local_image_ctx->name == m_image_state.name) { + image_locker.unlock(); + owner_locker.unlock(); + + update_features(); + return; + } + image_locker.unlock(); + + dout(15) << "local_image_name=" << m_local_image_ctx->name << ", " + << "remote_image_name=" << m_image_state.name << dendl; + + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_rename_image>(this); + m_local_image_ctx->operations->execute_rename(m_image_state.name, ctx); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_rename_image(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to rename image to '" << m_image_state.name << "': " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + update_features(); +} + +template <typename I> +void ApplyImageStateRequest<I>::update_features() { + uint64_t feature_updates = 0UL; + bool enabled = false; + + auto image_state_features = + m_image_state.features & ~RBD_FEATURES_IMPLICIT_ENABLE; + feature_updates = (m_features & ~image_state_features); + if (feature_updates == 0UL) { + feature_updates = (image_state_features & ~m_features); + enabled = (feature_updates != 0UL); + } + + if (feature_updates == 0UL) { + get_image_meta(); + return; + } + + dout(15) << "image_features=" << m_features << ", " + << "state_features=" << image_state_features << ", " + << "feature_updates=" << feature_updates << ", " + << "enabled=" << enabled << dendl; + + if (enabled) { + m_features |= feature_updates; + } else { + m_features &= ~feature_updates; + } + + std::shared_lock owner_lock{m_local_image_ctx->owner_lock}; + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_update_features>(this); + m_local_image_ctx->operations->execute_update_features( + feature_updates, enabled, ctx, 0U); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_update_features(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to update image features: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + update_features(); +} + +template <typename I> +void ApplyImageStateRequest<I>::get_image_meta() { + dout(15) << dendl; + + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_get_image_meta>(this); + auto req = librbd::image::GetMetadataRequest<I>::create( + m_local_image_ctx->md_ctx, m_local_image_ctx->header_oid, true, "", "", 0U, + &m_metadata, ctx); + req->send(); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_get_image_meta(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to fetch local image metadata: " << cpp_strerror(r) + << dendl; + finish(r); + return; + } + + update_image_meta(); +} + +template <typename I> +void ApplyImageStateRequest<I>::update_image_meta() { + std::set<std::string> keys_to_remove; + for (const auto& [key, value] : m_metadata) { + if (m_image_state.metadata.count(key) == 0) { + dout(15) << "removing image-meta key '" << key << "'" << dendl; + keys_to_remove.insert(key); + } + } + + std::map<std::string, bufferlist> metadata_to_update; + for (const auto& [key, value] : m_image_state.metadata) { + auto it = m_metadata.find(key); + if (it == m_metadata.end() || !it->second.contents_equal(value)) { + dout(15) << "updating image-meta key '" << key << "'" << dendl; + metadata_to_update.insert({key, value}); + } + } + + if (keys_to_remove.empty() && metadata_to_update.empty()) { + unprotect_snapshot(); + return; + } + + dout(15) << dendl; + + librados::ObjectWriteOperation op; + for (const auto& key : keys_to_remove) { + librbd::cls_client::metadata_remove(&op, key); + } + if (!metadata_to_update.empty()) { + librbd::cls_client::metadata_set(&op, metadata_to_update); + } + + auto aio_comp = create_rados_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_update_image_meta>(this); + int r = m_local_image_ctx->md_ctx.aio_operate(m_local_image_ctx->header_oid, aio_comp, + &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_update_image_meta(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to update image metadata: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_metadata.clear(); + + m_prev_snap_id = CEPH_NOSNAP; + unprotect_snapshot(); +} + +template <typename I> +void ApplyImageStateRequest<I>::unprotect_snapshot() { + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + + auto snap_it = m_local_image_ctx->snap_info.begin(); + if (m_prev_snap_id != CEPH_NOSNAP) { + snap_it = m_local_image_ctx->snap_info.upper_bound(m_prev_snap_id); + } + + for (; snap_it != m_local_image_ctx->snap_info.end(); ++snap_it) { + auto snap_id = snap_it->first; + const auto& snap_info = snap_it->second; + + auto user_ns = boost::get<cls::rbd::UserSnapshotNamespace>( + &snap_info.snap_namespace); + if (user_ns == nullptr) { + dout(20) << "snapshot " << snap_id << " is not a user snapshot" << dendl; + continue; + } + + if (snap_info.protection_status == RBD_PROTECTION_STATUS_UNPROTECTED) { + dout(20) << "snapshot " << snap_id << " is already unprotected" << dendl; + continue; + } + + auto snap_id_map_it = m_local_to_remote_snap_ids.find(snap_id); + if (snap_id_map_it == m_local_to_remote_snap_ids.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image" + << dendl; + break; + } + + auto remote_snap_id = snap_id_map_it->second; + auto snap_state_it = m_image_state.snapshots.find(remote_snap_id); + if (snap_state_it == m_image_state.snapshots.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image " + << "state" << dendl; + break; + } + + const auto& snap_state = snap_state_it->second; + if (snap_state.protection_status == RBD_PROTECTION_STATUS_UNPROTECTED) { + dout(15) << "snapshot " << snap_id << " is unprotected in remote image" + << dendl; + break; + } + } + + if (snap_it == m_local_image_ctx->snap_info.end()) { + image_locker.unlock(); + + // no local snapshots to unprotect + m_prev_snap_id = CEPH_NOSNAP; + remove_snapshot(); + return; + } + + m_prev_snap_id = snap_it->first; + m_snap_name = snap_it->second.name; + image_locker.unlock(); + + dout(15) << "snap_name=" << m_snap_name << ", " + << "snap_id=" << m_prev_snap_id << dendl; + + std::shared_lock owner_locker{m_local_image_ctx->owner_lock}; + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_unprotect_snapshot>(this); + m_local_image_ctx->operations->execute_snap_unprotect( + cls::rbd::UserSnapshotNamespace{}, m_snap_name.c_str(), ctx); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_unprotect_snapshot(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to unprotect snapshot " << m_snap_name << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + unprotect_snapshot(); +} + +template <typename I> +void ApplyImageStateRequest<I>::remove_snapshot() { + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + + auto snap_it = m_local_image_ctx->snap_info.begin(); + if (m_prev_snap_id != CEPH_NOSNAP) { + snap_it = m_local_image_ctx->snap_info.upper_bound(m_prev_snap_id); + } + + for (; snap_it != m_local_image_ctx->snap_info.end(); ++snap_it) { + auto snap_id = snap_it->first; + const auto& snap_info = snap_it->second; + + auto user_ns = boost::get<cls::rbd::UserSnapshotNamespace>( + &snap_info.snap_namespace); + if (user_ns == nullptr) { + dout(20) << "snapshot " << snap_id << " is not a user snapshot" << dendl; + continue; + } + + auto snap_id_map_it = m_local_to_remote_snap_ids.find(snap_id); + if (snap_id_map_it == m_local_to_remote_snap_ids.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image" + << dendl; + break; + } + + auto remote_snap_id = snap_id_map_it->second; + auto snap_state_it = m_image_state.snapshots.find(remote_snap_id); + if (snap_state_it == m_image_state.snapshots.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image " + << "state" << dendl; + break; + } + } + + if (snap_it == m_local_image_ctx->snap_info.end()) { + image_locker.unlock(); + + // no local snapshots to remove + m_prev_snap_id = CEPH_NOSNAP; + protect_snapshot(); + return; + } + + m_prev_snap_id = snap_it->first; + m_snap_name = snap_it->second.name; + image_locker.unlock(); + + dout(15) << "snap_name=" << m_snap_name << ", " + << "snap_id=" << m_prev_snap_id << dendl; + + std::shared_lock owner_locker{m_local_image_ctx->owner_lock}; + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_remove_snapshot>(this); + m_local_image_ctx->operations->execute_snap_remove( + cls::rbd::UserSnapshotNamespace{}, m_snap_name.c_str(), ctx); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_remove_snapshot(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to remove snapshot " << m_snap_name << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + remove_snapshot(); +} + +template <typename I> +void ApplyImageStateRequest<I>::protect_snapshot() { + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + + auto snap_it = m_local_image_ctx->snap_info.begin(); + if (m_prev_snap_id != CEPH_NOSNAP) { + snap_it = m_local_image_ctx->snap_info.upper_bound(m_prev_snap_id); + } + + for (; snap_it != m_local_image_ctx->snap_info.end(); ++snap_it) { + auto snap_id = snap_it->first; + const auto& snap_info = snap_it->second; + + auto user_ns = boost::get<cls::rbd::UserSnapshotNamespace>( + &snap_info.snap_namespace); + if (user_ns == nullptr) { + dout(20) << "snapshot " << snap_id << " is not a user snapshot" << dendl; + continue; + } + + if (snap_info.protection_status == RBD_PROTECTION_STATUS_PROTECTED) { + dout(20) << "snapshot " << snap_id << " is already protected" << dendl; + continue; + } + + auto snap_id_map_it = m_local_to_remote_snap_ids.find(snap_id); + if (snap_id_map_it == m_local_to_remote_snap_ids.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image" + << dendl; + continue; + } + + auto remote_snap_id = snap_id_map_it->second; + auto snap_state_it = m_image_state.snapshots.find(remote_snap_id); + if (snap_state_it == m_image_state.snapshots.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image " + << "state" << dendl; + continue; + } + + const auto& snap_state = snap_state_it->second; + if (snap_state.protection_status == RBD_PROTECTION_STATUS_PROTECTED) { + dout(15) << "snapshot " << snap_id << " is protected in remote image" + << dendl; + break; + } + } + + if (snap_it == m_local_image_ctx->snap_info.end()) { + image_locker.unlock(); + + // no local snapshots to protect + m_prev_snap_id = CEPH_NOSNAP; + rename_snapshot(); + return; + } + + m_prev_snap_id = snap_it->first; + m_snap_name = snap_it->second.name; + image_locker.unlock(); + + dout(15) << "snap_name=" << m_snap_name << ", " + << "snap_id=" << m_prev_snap_id << dendl; + + std::shared_lock owner_locker{m_local_image_ctx->owner_lock}; + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_protect_snapshot>(this); + m_local_image_ctx->operations->execute_snap_protect( + cls::rbd::UserSnapshotNamespace{}, m_snap_name.c_str(), ctx); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_protect_snapshot(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to protect snapshot " << m_snap_name << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + protect_snapshot(); +} + +template <typename I> +void ApplyImageStateRequest<I>::rename_snapshot() { + std::shared_lock image_locker{m_local_image_ctx->image_lock}; + + auto snap_it = m_local_image_ctx->snap_info.begin(); + if (m_prev_snap_id != CEPH_NOSNAP) { + snap_it = m_local_image_ctx->snap_info.upper_bound(m_prev_snap_id); + } + + for (; snap_it != m_local_image_ctx->snap_info.end(); ++snap_it) { + auto snap_id = snap_it->first; + const auto& snap_info = snap_it->second; + + auto user_ns = boost::get<cls::rbd::UserSnapshotNamespace>( + &snap_info.snap_namespace); + if (user_ns == nullptr) { + dout(20) << "snapshot " << snap_id << " is not a user snapshot" << dendl; + continue; + } + + auto snap_id_map_it = m_local_to_remote_snap_ids.find(snap_id); + if (snap_id_map_it == m_local_to_remote_snap_ids.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image" + << dendl; + continue; + } + + auto remote_snap_id = snap_id_map_it->second; + auto snap_state_it = m_image_state.snapshots.find(remote_snap_id); + if (snap_state_it == m_image_state.snapshots.end()) { + dout(15) << "snapshot " << snap_id << " does not exist in remote image " + << "state" << dendl; + continue; + } + + const auto& snap_state = snap_state_it->second; + if (snap_info.name != snap_state.name) { + dout(15) << "snapshot " << snap_id << " has been renamed from '" + << snap_info.name << "' to '" << snap_state.name << "'" + << dendl; + m_snap_name = snap_state.name; + break; + } + } + + if (snap_it == m_local_image_ctx->snap_info.end()) { + image_locker.unlock(); + + // no local snapshots to protect + m_prev_snap_id = CEPH_NOSNAP; + set_snapshot_limit(); + return; + } + + m_prev_snap_id = snap_it->first; + image_locker.unlock(); + + dout(15) << "snap_name=" << m_snap_name << ", " + << "snap_id=" << m_prev_snap_id << dendl; + + std::shared_lock owner_locker{m_local_image_ctx->owner_lock}; + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_rename_snapshot>(this); + m_local_image_ctx->operations->execute_snap_rename( + m_prev_snap_id, m_snap_name.c_str(), ctx); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_rename_snapshot(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to protect snapshot " << m_snap_name << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + rename_snapshot(); +} + +template <typename I> +void ApplyImageStateRequest<I>::set_snapshot_limit() { + dout(15) << "snap_limit=" << m_image_state.snap_limit << dendl; + + // no need to even check the current limit -- just set it + std::shared_lock owner_locker{m_local_image_ctx->owner_lock}; + auto ctx = create_context_callback< + ApplyImageStateRequest<I>, + &ApplyImageStateRequest<I>::handle_set_snapshot_limit>(this); + m_local_image_ctx->operations->execute_snap_set_limit( + m_image_state.snap_limit, ctx); +} + +template <typename I> +void ApplyImageStateRequest<I>::handle_set_snapshot_limit(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to update snapshot limit: " << cpp_strerror(r) + << dendl; + } + + finish(r); +} + +template <typename I> +void ApplyImageStateRequest<I>::finish(int r) { + dout(15) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +template <typename I> +uint64_t ApplyImageStateRequest<I>::compute_remote_snap_id( + uint64_t local_snap_id) { + ceph_assert(ceph_mutex_is_locked(m_local_image_ctx->image_lock)); + ceph_assert(ceph_mutex_is_locked(m_remote_image_ctx->image_lock)); + + // Search our local non-primary snapshots for a mapping to the remote + // snapshot. The non-primary mirror snapshot with the mappings will always + // come at or after the snapshot we are searching against + auto remote_snap_id = util::compute_remote_snap_id( + m_local_image_ctx->image_lock, m_local_image_ctx->snap_info, + local_snap_id, m_remote_mirror_uuid); + if (remote_snap_id != CEPH_NOSNAP) { + return remote_snap_id; + } + + // if we failed to find a match to a remote snapshot in our local non-primary + // snapshots, check the remote image for non-primary snapshot mappings back + // to our snapshot + for (auto snap_it = m_remote_image_ctx->snap_info.begin(); + snap_it != m_remote_image_ctx->snap_info.end(); ++snap_it) { + auto snap_id = snap_it->first; + auto mirror_ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &snap_it->second.snap_namespace); + if (mirror_ns == nullptr || !mirror_ns->is_non_primary()) { + continue; + } + + if (mirror_ns->primary_mirror_uuid != m_local_mirror_uuid) { + dout(20) << "remote snapshot " << snap_id << " not tied to local" + << dendl; + continue; + } else if (mirror_ns->primary_snap_id == local_snap_id) { + dout(15) << "local snapshot " << local_snap_id << " maps to " + << "remote snapshot " << snap_id << dendl; + return snap_id; + } + + const auto& snap_seqs = mirror_ns->snap_seqs; + for (auto [local_snap_id_seq, remote_snap_id_seq] : snap_seqs) { + if (local_snap_id_seq == local_snap_id) { + dout(15) << "local snapshot " << local_snap_id << " maps to " + << "remote snapshot " << remote_snap_id_seq << dendl; + return remote_snap_id_seq; + } + } + } + + return CEPH_NOSNAP; +} + +template <typename I> +void ApplyImageStateRequest<I>::compute_local_to_remote_snap_ids() { + ceph_assert(ceph_mutex_is_locked(m_local_image_ctx->image_lock)); + std::shared_lock remote_image_locker{m_remote_image_ctx->image_lock}; + + for (const auto& [snap_id, snap_info] : m_local_image_ctx->snap_info) { + m_local_to_remote_snap_ids[snap_id] = compute_remote_snap_id(snap_id); + } + + dout(15) << "local_to_remote_snap_ids=" << m_local_to_remote_snap_ids + << dendl; +} + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::snapshot::ApplyImageStateRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h b/src/tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h new file mode 100644 index 000000000..0e2d09ddf --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h @@ -0,0 +1,155 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_APPLY_IMAGE_STATE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_APPLY_IMAGE_STATE_REQUEST_H + +#include "common/ceph_mutex.h" +#include "librbd/mirror/snapshot/Types.h" +#include <map> +#include <string> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +template <typename> class EventPreprocessor; +template <typename> class ReplayStatusFormatter; +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class ApplyImageStateRequest { +public: + static ApplyImageStateRequest* create( + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + ImageCtxT* local_image_ctx, + ImageCtxT* remote_image_ctx, + librbd::mirror::snapshot::ImageState image_state, + Context* on_finish) { + return new ApplyImageStateRequest(local_mirror_uuid, remote_mirror_uuid, + local_image_ctx, remote_image_ctx, + image_state, on_finish); + } + + ApplyImageStateRequest( + const std::string& local_mirror_uuid, + const std::string& remote_mirror_uuid, + ImageCtxT* local_image_ctx, + ImageCtxT* remote_image_ctx, + librbd::mirror::snapshot::ImageState image_state, + Context* on_finish); + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * RENAME_IMAGE + * | + * | /---------\ + * | | | + * v v | + * UPDATE_FEATURES -----/ + * | + * v + * GET_IMAGE_META + * | + * | /---------\ + * | | | + * v v | + * UPDATE_IMAGE_META ---/ + * | + * | /---------\ + * | | | + * v v | + * UNPROTECT_SNAPSHOT | + * | | + * v | + * REMOVE_SNAPSHOT | + * | | + * v | + * PROTECT_SNAPSHOT | + * | | + * v | + * RENAME_SNAPSHOT -----/ + * | + * v + * SET_SNAPSHOT_LIMIT + * | + * v + * <finish> + * + * @endverbatim + */ + + std::string m_local_mirror_uuid; + std::string m_remote_mirror_uuid; + ImageCtxT* m_local_image_ctx; + ImageCtxT* m_remote_image_ctx; + librbd::mirror::snapshot::ImageState m_image_state; + Context* m_on_finish; + + std::map<uint64_t, uint64_t> m_local_to_remote_snap_ids; + + uint64_t m_features = 0; + + std::map<std::string, bufferlist> m_metadata; + + uint64_t m_prev_snap_id = 0; + std::string m_snap_name; + + void rename_image(); + void handle_rename_image(int r); + + void update_features(); + void handle_update_features(int r); + + void get_image_meta(); + void handle_get_image_meta(int r); + + void update_image_meta(); + void handle_update_image_meta(int r); + + void unprotect_snapshot(); + void handle_unprotect_snapshot(int r); + + void remove_snapshot(); + void handle_remove_snapshot(int r); + + void protect_snapshot(); + void handle_protect_snapshot(int r); + + void rename_snapshot(); + void handle_rename_snapshot(int r); + + void set_snapshot_limit(); + void handle_set_snapshot_limit(int r); + + void finish(int r); + + uint64_t compute_remote_snap_id(uint64_t snap_id); + void compute_local_to_remote_snap_ids(); +}; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::snapshot::ApplyImageStateRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_APPLY_IMAGE_STATE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc new file mode 100644 index 000000000..c923395c9 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.cc @@ -0,0 +1,204 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "CreateLocalImageRequest.h" +#include "include/rados/librados.hpp" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_client.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/image_replayer/CreateImageRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::snapshot::" \ + << "CreateLocalImageRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +void CreateLocalImageRequest<I>::send() { + disable_mirror_image(); +} + +template <typename I> +void CreateLocalImageRequest<I>::disable_mirror_image() { + if (m_state_builder->local_image_id.empty()) { + add_mirror_image(); + return; + } + + dout(10) << dendl; + update_progress("DISABLE_MIRROR_IMAGE"); + + // need to send 'disabling' since the cls methods will fail if we aren't + // in that state + cls::rbd::MirrorImage mirror_image{ + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}; + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_set(&op, m_state_builder->local_image_id, + mirror_image); + + auto aio_comp = create_rados_callback< + CreateLocalImageRequest<I>, + &CreateLocalImageRequest<I>::handle_disable_mirror_image>(this); + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void CreateLocalImageRequest<I>::handle_disable_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to disable mirror image " << m_global_image_id << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + remove_mirror_image(); +} + +template <typename I> +void CreateLocalImageRequest<I>::remove_mirror_image() { + dout(10) << dendl; + update_progress("REMOVE_MIRROR_IMAGE"); + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_remove(&op, m_state_builder->local_image_id); + + auto aio_comp = create_rados_callback< + CreateLocalImageRequest<I>, + &CreateLocalImageRequest<I>::handle_remove_mirror_image>(this); + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void CreateLocalImageRequest<I>::handle_remove_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to remove mirror image " << m_global_image_id << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_state_builder->local_image_id = ""; + add_mirror_image(); +} + +template <typename I> +void CreateLocalImageRequest<I>::add_mirror_image() { + ceph_assert(m_state_builder->local_image_id.empty()); + m_state_builder->local_image_id = + librbd::util::generate_image_id<I>(m_local_io_ctx); + + dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl; + update_progress("ADD_MIRROR_IMAGE"); + + // use 'creating' to track a partially constructed image. it will + // be switched to 'enabled' once the image is fully created + cls::rbd::MirrorImage mirror_image{ + cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, m_global_image_id, + cls::rbd::MIRROR_IMAGE_STATE_CREATING}; + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_set(&op, m_state_builder->local_image_id, + mirror_image); + + auto aio_comp = create_rados_callback< + CreateLocalImageRequest<I>, + &CreateLocalImageRequest<I>::handle_add_mirror_image>(this); + int r = m_local_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void CreateLocalImageRequest<I>::handle_add_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to register mirror image " << m_global_image_id << ": " + << cpp_strerror(r) << dendl; + this->finish(r); + return; + } + + create_local_image(); +} + +template <typename I> +void CreateLocalImageRequest<I>::create_local_image() { + dout(10) << "local_image_id=" << m_state_builder->local_image_id << dendl; + update_progress("CREATE_LOCAL_IMAGE"); + + m_remote_image_ctx->image_lock.lock_shared(); + std::string image_name = m_remote_image_ctx->name; + m_remote_image_ctx->image_lock.unlock_shared(); + + auto ctx = create_context_callback< + CreateLocalImageRequest<I>, + &CreateLocalImageRequest<I>::handle_create_local_image>(this); + auto request = CreateImageRequest<I>::create( + m_threads, m_local_io_ctx, m_global_image_id, + m_state_builder->remote_mirror_uuid, image_name, + m_state_builder->local_image_id, m_remote_image_ctx, + m_pool_meta_cache, cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, ctx); + request->send(); +} +template <typename I> +void CreateLocalImageRequest<I>::handle_create_local_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -EBADF) { + dout(5) << "image id " << m_state_builder->local_image_id << " " + << "already in-use" << dendl; + disable_mirror_image(); + return; + } else if (r < 0) { + if (r == -ENOENT) { + dout(10) << "parent image does not exist" << dendl; + } else { + derr << "failed to create local image: " << cpp_strerror(r) << dendl; + } + finish(r); + return; + } + + finish(0); +} + +template <typename I> +void CreateLocalImageRequest<I>::update_progress( + const std::string& description) { + dout(15) << description << dendl; + if (m_progress_ctx != nullptr) { + m_progress_ctx->update_progress(description); + } +} + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::snapshot::CreateLocalImageRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h new file mode 100644 index 000000000..3345154b4 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h @@ -0,0 +1,121 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_CREATE_LOCAL_IMAGE_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_CREATE_LOCAL_IMAGE_REQUEST_H + +#include "include/rados/librados_fwd.hpp" +#include "tools/rbd_mirror/BaseRequest.h" +#include <string> + +struct Context; +namespace librbd { class ImageCtx; } + +namespace rbd { +namespace mirror { + +class PoolMetaCache; +class ProgressContext; +template <typename> struct Threads; + +namespace image_replayer { +namespace snapshot { + +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class CreateLocalImageRequest : public BaseRequest { +public: + typedef rbd::mirror::ProgressContext ProgressContext; + + static CreateLocalImageRequest* create( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + ImageCtxT* remote_image_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + Context* on_finish) { + return new CreateLocalImageRequest(threads, local_io_ctx, remote_image_ctx, + global_image_id, pool_meta_cache, + progress_ctx, state_builder, on_finish); + } + + CreateLocalImageRequest( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + ImageCtxT* remote_image_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + Context* on_finish) + : BaseRequest(on_finish), + m_threads(threads), + m_local_io_ctx(local_io_ctx), + m_remote_image_ctx(remote_image_ctx), + m_global_image_id(global_image_id), + m_pool_meta_cache(pool_meta_cache), + m_progress_ctx(progress_ctx), + m_state_builder(state_builder) { + } + + void send(); + +private: + /** + * @verbatim + * + * <start> + * | + * v + * DISABLE_MIRROR_IMAGE < * * * * * * + * | * + * v * + * REMOVE_MIRROR_IMAGE * + * | * + * v * + * ADD_MIRROR_IMAGE * + * | * + * v (id exists) * + * CREATE_LOCAL_IMAGE * * * * * * * * + * | + * v + * <finish> + * + * @endverbatim + */ + + Threads<ImageCtxT>* m_threads; + librados::IoCtx& m_local_io_ctx; + ImageCtxT* m_remote_image_ctx; + std::string m_global_image_id; + PoolMetaCache* m_pool_meta_cache; + ProgressContext* m_progress_ctx; + StateBuilder<ImageCtxT>* m_state_builder; + + void disable_mirror_image(); + void handle_disable_mirror_image(int r); + + void remove_mirror_image(); + void handle_remove_mirror_image(int r); + + void add_mirror_image(); + void handle_add_mirror_image(int r); + + void create_local_image(); + void handle_create_local_image(int r); + + void update_progress(const std::string& description); + +}; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::snapshot::CreateLocalImageRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_CREATE_LOCAL_IMAGE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/PrepareReplayRequest.cc b/src/tools/rbd_mirror/image_replayer/snapshot/PrepareReplayRequest.cc new file mode 100644 index 000000000..575eb8534 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/PrepareReplayRequest.cc @@ -0,0 +1,70 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "PrepareReplayRequest.h" +#include "common/debug.h" +#include "common/dout.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "tools/rbd_mirror/ProgressContext.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::snapshot::" \ + << "PrepareReplayRequest: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +using librbd::util::create_context_callback; + +template <typename I> +void PrepareReplayRequest<I>::send() { + *m_resync_requested = false; + *m_syncing = false; + + load_local_image_meta(); +} + +template <typename I> +void PrepareReplayRequest<I>::load_local_image_meta() { + dout(15) << dendl; + + ceph_assert(m_state_builder->local_image_meta == nullptr); + m_state_builder->local_image_meta = + librbd::mirror::snapshot::ImageMeta<I>::create( + m_state_builder->local_image_ctx, m_local_mirror_uuid); + + auto ctx = create_context_callback< + PrepareReplayRequest<I>, + &PrepareReplayRequest<I>::handle_load_local_image_meta>(this); + m_state_builder->local_image_meta->load(ctx); +} + +template <typename I> +void PrepareReplayRequest<I>::handle_load_local_image_meta(int r) { + dout(15) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "failed to load local image-meta: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + *m_resync_requested = m_state_builder->local_image_meta->resync_requested; + finish(0); +} + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::snapshot::PrepareReplayRequest<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/PrepareReplayRequest.h b/src/tools/rbd_mirror/image_replayer/snapshot/PrepareReplayRequest.h new file mode 100644 index 000000000..4e9246acd --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/PrepareReplayRequest.h @@ -0,0 +1,92 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H +#define RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H + +#include "include/int_types.h" +#include "librbd/mirror/Types.h" +#include "tools/rbd_mirror/BaseRequest.h" +#include <list> +#include <string> + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { + +class ProgressContext; + +namespace image_replayer { +namespace snapshot { + +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class PrepareReplayRequest : public BaseRequest { +public: + static PrepareReplayRequest* create( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + bool* resync_requested, + bool* syncing, + Context* on_finish) { + return new PrepareReplayRequest( + local_mirror_uuid, progress_ctx, state_builder, resync_requested, + syncing, on_finish); + } + + PrepareReplayRequest( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + StateBuilder<ImageCtxT>* state_builder, + bool* resync_requested, + bool* syncing, + Context* on_finish) + : BaseRequest(on_finish), + m_local_mirror_uuid(local_mirror_uuid), + m_progress_ctx(progress_ctx), + m_state_builder(state_builder), + m_resync_requested(resync_requested), + m_syncing(syncing) { + } + + void send() override; + +private: + // TODO + /** + * @verbatim + * + * <start> + * | + * v + * LOAD_LOCAL_IMAGE_META + * | + * v + * <finish> + * + * @endverbatim + */ + + std::string m_local_mirror_uuid; + ProgressContext* m_progress_ctx; + StateBuilder<ImageCtxT>* m_state_builder; + bool* m_resync_requested; + bool* m_syncing; + + void load_local_image_meta(); + void handle_load_local_image_meta(int r); + +}; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::snapshot::PrepareReplayRequest<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_JOURNAL_PREPARE_REPLAY_REQUEST_H diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc new file mode 100644 index 000000000..4a44a57bc --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc @@ -0,0 +1,1586 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "Replayer.h" +#include "common/debug.h" +#include "common/errno.h" +#include "include/stringify.h" +#include "common/Timer.h" +#include "cls/rbd/cls_rbd_client.h" +#include "json_spirit/json_spirit.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "librbd/deep_copy/Handler.h" +#include "librbd/deep_copy/ImageCopyRequest.h" +#include "librbd/deep_copy/SnapshotCopyRequest.h" +#include "librbd/mirror/ImageStateUpdateRequest.h" +#include "librbd/mirror/snapshot/CreateNonPrimaryRequest.h" +#include "librbd/mirror/snapshot/GetImageStateRequest.h" +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "tools/rbd_mirror/InstanceWatcher.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" +#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" +#include "tools/rbd_mirror/image_replayer/ReplayerListener.h" +#include "tools/rbd_mirror/image_replayer/Utils.h" +#include "tools/rbd_mirror/image_replayer/snapshot/ApplyImageStateRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h" +#include "tools/rbd_mirror/image_replayer/snapshot/Utils.h" +#include <set> + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::snapshot::" \ + << "Replayer: " << this << " " << __func__ << ": " + +extern PerfCounters *g_snapshot_perf_counters; + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +namespace { + +double round_to_two_places(double value) { + return abs(round(value * 100) / 100); +} + +template<typename I> +std::pair<uint64_t, librbd::SnapInfo*> get_newest_mirror_snapshot( + I* image_ctx) { + for (auto snap_info_it = image_ctx->snap_info.rbegin(); + snap_info_it != image_ctx->snap_info.rend(); ++snap_info_it) { + const auto& snap_ns = snap_info_it->second.snap_namespace; + auto mirror_ns = boost::get< + cls::rbd::MirrorSnapshotNamespace>(&snap_ns); + if (mirror_ns == nullptr || !mirror_ns->complete) { + continue; + } + + return {snap_info_it->first, &snap_info_it->second}; + } + + return {CEPH_NOSNAP, nullptr}; +} + +} // anonymous namespace + +using librbd::util::create_async_context_callback; +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template <typename I> +struct Replayer<I>::C_UpdateWatchCtx : public librbd::UpdateWatchCtx { + Replayer<I>* replayer; + + C_UpdateWatchCtx(Replayer<I>* replayer) : replayer(replayer) { + } + + void handle_notify() override { + replayer->handle_image_update_notify(); + } +}; + +template <typename I> +struct Replayer<I>::DeepCopyHandler : public librbd::deep_copy::Handler { + Replayer *replayer; + + DeepCopyHandler(Replayer* replayer) : replayer(replayer) { + } + + void handle_read(uint64_t bytes_read) override { + replayer->handle_copy_image_read(bytes_read); + } + + int update_progress(uint64_t object_number, uint64_t object_count) override { + replayer->handle_copy_image_progress(object_number, object_count); + return 0; + } +}; + +template <typename I> +Replayer<I>::Replayer( + Threads<I>* threads, + InstanceWatcher<I>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + StateBuilder<I>* state_builder, + ReplayerListener* replayer_listener) + : m_threads(threads), + m_instance_watcher(instance_watcher), + m_local_mirror_uuid(local_mirror_uuid), + m_pool_meta_cache(pool_meta_cache), + m_state_builder(state_builder), + m_replayer_listener(replayer_listener), + m_lock(ceph::make_mutex(librbd::util::unique_lock_name( + "rbd::mirror::image_replayer::snapshot::Replayer", this))) { + dout(10) << dendl; +} + +template <typename I> +Replayer<I>::~Replayer() { + dout(10) << dendl; + + { + std::unique_lock locker{m_lock}; + unregister_perf_counters(); + } + + ceph_assert(m_state == STATE_COMPLETE); + ceph_assert(m_update_watch_ctx == nullptr); + ceph_assert(m_deep_copy_handler == nullptr); +} + +template <typename I> +void Replayer<I>::init(Context* on_finish) { + dout(10) << dendl; + + ceph_assert(m_state == STATE_INIT); + + RemotePoolMeta remote_pool_meta; + int r = m_pool_meta_cache->get_remote_pool_meta( + m_state_builder->remote_image_ctx->md_ctx.get_id(), &remote_pool_meta); + if (r < 0 || remote_pool_meta.mirror_peer_uuid.empty()) { + derr << "failed to retrieve mirror peer uuid from remote pool" << dendl; + m_state = STATE_COMPLETE; + m_threads->work_queue->queue(on_finish, r); + return; + } + + m_remote_mirror_peer_uuid = remote_pool_meta.mirror_peer_uuid; + dout(10) << "remote_mirror_peer_uuid=" << m_remote_mirror_peer_uuid << dendl; + + { + auto local_image_ctx = m_state_builder->local_image_ctx; + std::shared_lock image_locker{local_image_ctx->image_lock}; + m_image_spec = image_replayer::util::compute_image_spec( + local_image_ctx->md_ctx, local_image_ctx->name); + } + + { + std::unique_lock locker{m_lock}; + register_perf_counters(); + } + + ceph_assert(m_on_init_shutdown == nullptr); + m_on_init_shutdown = on_finish; + + register_local_update_watcher(); +} + +template <typename I> +void Replayer<I>::shut_down(Context* on_finish) { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + ceph_assert(m_on_init_shutdown == nullptr); + m_on_init_shutdown = on_finish; + m_error_code = 0; + m_error_description = ""; + + ceph_assert(m_state != STATE_INIT); + auto state = STATE_COMPLETE; + std::swap(m_state, state); + + if (state == STATE_REPLAYING) { + // if a sync request was pending, request a cancelation + m_instance_watcher->cancel_sync_request( + m_state_builder->local_image_ctx->id); + + // TODO interrupt snapshot copy and image copy state machines even if remote + // cluster is unreachable + dout(10) << "shut down pending on completion of snapshot replay" << dendl; + return; + } + locker.unlock(); + + unregister_remote_update_watcher(); +} + +template <typename I> +void Replayer<I>::flush(Context* on_finish) { + dout(10) << dendl; + + // TODO + m_threads->work_queue->queue(on_finish, 0); +} + +template <typename I> +bool Replayer<I>::get_replay_status(std::string* description, + Context* on_finish) { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + if (m_state != STATE_REPLAYING && m_state != STATE_IDLE) { + locker.unlock(); + + derr << "replay not running" << dendl; + on_finish->complete(-EAGAIN); + return false; + } + + std::shared_lock local_image_locker{ + m_state_builder->local_image_ctx->image_lock}; + auto [local_snap_id, local_snap_info] = get_newest_mirror_snapshot( + m_state_builder->local_image_ctx); + + std::shared_lock remote_image_locker{ + m_state_builder->remote_image_ctx->image_lock}; + auto [remote_snap_id, remote_snap_info] = get_newest_mirror_snapshot( + m_state_builder->remote_image_ctx); + + if (remote_snap_info == nullptr) { + remote_image_locker.unlock(); + local_image_locker.unlock(); + locker.unlock(); + + derr << "remote image does not contain mirror snapshots" << dendl; + on_finish->complete(-EAGAIN); + return false; + } + + std::string replay_state = "idle"; + if (m_remote_snap_id_end != CEPH_NOSNAP) { + replay_state = "syncing"; + } + + json_spirit::mObject root_obj; + root_obj["replay_state"] = replay_state; + root_obj["remote_snapshot_timestamp"] = remote_snap_info->timestamp.sec(); + + auto matching_remote_snap_id = util::compute_remote_snap_id( + m_state_builder->local_image_ctx->image_lock, + m_state_builder->local_image_ctx->snap_info, + local_snap_id, m_state_builder->remote_mirror_uuid); + auto matching_remote_snap_it = + m_state_builder->remote_image_ctx->snap_info.find(matching_remote_snap_id); + if (matching_remote_snap_id != CEPH_NOSNAP && + matching_remote_snap_it != + m_state_builder->remote_image_ctx->snap_info.end()) { + // use the timestamp from the matching remote image since + // the local snapshot would just be the time the snapshot was + // synced and not the consistency point in time. + root_obj["local_snapshot_timestamp"] = + matching_remote_snap_it->second.timestamp.sec(); + } + + matching_remote_snap_it = m_state_builder->remote_image_ctx->snap_info.find( + m_remote_snap_id_end); + if (m_remote_snap_id_end != CEPH_NOSNAP && + matching_remote_snap_it != + m_state_builder->remote_image_ctx->snap_info.end()) { + root_obj["syncing_snapshot_timestamp"] = remote_snap_info->timestamp.sec(); + root_obj["syncing_percent"] = static_cast<uint64_t>( + 100 * m_local_mirror_snap_ns.last_copied_object_number / + static_cast<float>(std::max<uint64_t>(1U, m_local_object_count))); + } + + m_bytes_per_second(0); + auto bytes_per_second = m_bytes_per_second.get_average(); + root_obj["bytes_per_second"] = round_to_two_places(bytes_per_second); + + auto bytes_per_snapshot = boost::accumulators::rolling_mean( + m_bytes_per_snapshot); + root_obj["bytes_per_snapshot"] = round_to_two_places(bytes_per_snapshot); + + auto pending_bytes = bytes_per_snapshot * m_pending_snapshots; + if (bytes_per_second > 0 && m_pending_snapshots > 0) { + std::uint64_t seconds_until_synced = round_to_two_places( + pending_bytes / bytes_per_second); + if (seconds_until_synced >= std::numeric_limits<uint64_t>::max()) { + seconds_until_synced = std::numeric_limits<uint64_t>::max(); + } + + root_obj["seconds_until_synced"] = seconds_until_synced; + } + + *description = json_spirit::write( + root_obj, json_spirit::remove_trailing_zeros); + + local_image_locker.unlock(); + remote_image_locker.unlock(); + locker.unlock(); + on_finish->complete(-EEXIST); + return true; +} + +template <typename I> +void Replayer<I>::load_local_image_meta() { + dout(10) << dendl; + + { + // reset state in case new snapshot is added while we are scanning + std::unique_lock locker{m_lock}; + m_image_updated = false; + } + + bool update_status = false; + { + auto local_image_ctx = m_state_builder->local_image_ctx; + std::shared_lock image_locker{local_image_ctx->image_lock}; + auto image_spec = image_replayer::util::compute_image_spec( + local_image_ctx->md_ctx, local_image_ctx->name); + if (m_image_spec != image_spec) { + m_image_spec = image_spec; + update_status = true; + } + } + if (update_status) { + std::unique_lock locker{m_lock}; + unregister_perf_counters(); + register_perf_counters(); + notify_status_updated(); + } + + ceph_assert(m_state_builder->local_image_meta != nullptr); + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_load_local_image_meta>(this); + m_state_builder->local_image_meta->load(ctx); +} + +template <typename I> +void Replayer<I>::handle_load_local_image_meta(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "failed to load local image-meta: " << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to load local image-meta"); + return; + } + + if (r >= 0 && m_state_builder->local_image_meta->resync_requested) { + m_resync_requested = true; + + dout(10) << "local image resync requested" << dendl; + handle_replay_complete(0, "resync requested"); + return; + } + + refresh_local_image(); +} + +template <typename I> +void Replayer<I>::refresh_local_image() { + if (!m_state_builder->local_image_ctx->state->is_refresh_required()) { + refresh_remote_image(); + return; + } + + dout(10) << dendl; + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_refresh_local_image>(this); + m_state_builder->local_image_ctx->state->refresh(ctx); +} + +template <typename I> +void Replayer<I>::handle_refresh_local_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to refresh local image: " << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to refresh local image"); + return; + } + + refresh_remote_image(); +} + +template <typename I> +void Replayer<I>::refresh_remote_image() { + if (!m_state_builder->remote_image_ctx->state->is_refresh_required()) { + std::unique_lock locker{m_lock}; + scan_local_mirror_snapshots(&locker); + return; + } + + dout(10) << dendl; + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_refresh_remote_image>(this); + m_state_builder->remote_image_ctx->state->refresh(ctx); +} + +template <typename I> +void Replayer<I>::handle_refresh_remote_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to refresh remote image: " << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to refresh remote image"); + return; + } + + std::unique_lock locker{m_lock}; + scan_local_mirror_snapshots(&locker); +} + +template <typename I> +void Replayer<I>::scan_local_mirror_snapshots( + std::unique_lock<ceph::mutex>* locker) { + if (is_replay_interrupted(locker)) { + return; + } + + dout(10) << dendl; + + m_local_snap_id_start = 0; + m_local_snap_id_end = CEPH_NOSNAP; + m_local_mirror_snap_ns = {}; + m_local_object_count = 0; + + m_remote_snap_id_start = 0; + m_remote_snap_id_end = CEPH_NOSNAP; + m_remote_mirror_snap_ns = {}; + + std::set<uint64_t> prune_snap_ids; + + auto local_image_ctx = m_state_builder->local_image_ctx; + std::shared_lock image_locker{local_image_ctx->image_lock}; + for (auto snap_info_it = local_image_ctx->snap_info.begin(); + snap_info_it != local_image_ctx->snap_info.end(); ++snap_info_it) { + const auto& snap_ns = snap_info_it->second.snap_namespace; + auto mirror_ns = boost::get< + cls::rbd::MirrorSnapshotNamespace>(&snap_ns); + if (mirror_ns == nullptr) { + continue; + } + + dout(15) << "local mirror snapshot: id=" << snap_info_it->first << ", " + << "mirror_ns=" << *mirror_ns << dendl; + m_local_mirror_snap_ns = *mirror_ns; + + auto local_snap_id = snap_info_it->first; + if (mirror_ns->is_non_primary()) { + if (mirror_ns->complete) { + // if remote has new snapshots, we would sync from here + m_local_snap_id_start = local_snap_id; + ceph_assert(m_local_snap_id_end == CEPH_NOSNAP); + + if (mirror_ns->mirror_peer_uuids.empty()) { + // no other peer will attempt to sync to this snapshot so store as + // a candidate for removal + prune_snap_ids.insert(local_snap_id); + } + } else if (mirror_ns->last_copied_object_number == 0 && + m_local_snap_id_start > 0) { + // snapshot might be missing image state, object-map, etc, so just + // delete and re-create it if we haven't started copying data + // objects. Also only prune this snapshot since we will need the + // previous mirror snapshot for syncing. Special case exception for + // the first non-primary snapshot since we know its snapshot is + // well-formed because otherwise the mirror-image-state would have + // forced an image deletion. + prune_snap_ids.clear(); + prune_snap_ids.insert(local_snap_id); + break; + } else { + // start snap will be last complete mirror snapshot or initial + // image revision + m_local_snap_id_end = local_snap_id; + break; + } + } else if (mirror_ns->is_primary()) { + if (mirror_ns->complete) { + m_local_snap_id_start = local_snap_id; + ceph_assert(m_local_snap_id_end == CEPH_NOSNAP); + } else { + derr << "incomplete local primary snapshot" << dendl; + handle_replay_complete(locker, -EINVAL, + "incomplete local primary snapshot"); + return; + } + } else { + derr << "unknown local mirror snapshot state" << dendl; + handle_replay_complete(locker, -EINVAL, + "invalid local mirror snapshot state"); + return; + } + } + image_locker.unlock(); + + if (m_local_snap_id_start > 0) { + // remove candidate that is required for delta snapshot sync + prune_snap_ids.erase(m_local_snap_id_start); + } + if (!prune_snap_ids.empty()) { + locker->unlock(); + + auto prune_snap_id = *prune_snap_ids.begin(); + dout(5) << "pruning unused non-primary snapshot " << prune_snap_id << dendl; + prune_non_primary_snapshot(prune_snap_id); + return; + } + + if (m_local_snap_id_start > 0 || m_local_snap_id_end != CEPH_NOSNAP) { + if (m_local_mirror_snap_ns.is_non_primary() && + m_local_mirror_snap_ns.primary_mirror_uuid != + m_state_builder->remote_mirror_uuid) { + // TODO support multiple peers + derr << "local image linked to unknown peer: " + << m_local_mirror_snap_ns.primary_mirror_uuid << dendl; + handle_replay_complete(locker, -EEXIST, + "local image linked to unknown peer"); + return; + } else if (m_local_mirror_snap_ns.state == + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY) { + dout(5) << "local image promoted" << dendl; + handle_replay_complete(locker, 0, "force promoted"); + return; + } + + dout(10) << "found local mirror snapshot: " + << "local_snap_id_start=" << m_local_snap_id_start << ", " + << "local_snap_id_end=" << m_local_snap_id_end << ", " + << "local_snap_ns=" << m_local_mirror_snap_ns << dendl; + if (!m_local_mirror_snap_ns.is_primary() && + m_local_mirror_snap_ns.complete) { + // our remote sync should start after this completed snapshot + m_remote_snap_id_start = m_local_mirror_snap_ns.primary_snap_id; + } + } + + // we don't have any mirror snapshots or only completed non-primary + // mirror snapshots + scan_remote_mirror_snapshots(locker); +} + +template <typename I> +void Replayer<I>::scan_remote_mirror_snapshots( + std::unique_lock<ceph::mutex>* locker) { + dout(10) << dendl; + + m_pending_snapshots = 0; + + std::set<uint64_t> unlink_snap_ids; + bool split_brain = false; + bool remote_demoted = false; + auto remote_image_ctx = m_state_builder->remote_image_ctx; + std::shared_lock image_locker{remote_image_ctx->image_lock}; + for (auto snap_info_it = remote_image_ctx->snap_info.begin(); + snap_info_it != remote_image_ctx->snap_info.end(); ++snap_info_it) { + const auto& snap_ns = snap_info_it->second.snap_namespace; + auto mirror_ns = boost::get< + cls::rbd::MirrorSnapshotNamespace>(&snap_ns); + if (mirror_ns == nullptr) { + continue; + } + + dout(15) << "remote mirror snapshot: id=" << snap_info_it->first << ", " + << "mirror_ns=" << *mirror_ns << dendl; + remote_demoted = mirror_ns->is_demoted(); + if (!mirror_ns->is_primary() && !mirror_ns->is_non_primary()) { + derr << "unknown remote mirror snapshot state" << dendl; + handle_replay_complete(locker, -EINVAL, + "invalid remote mirror snapshot state"); + return; + } else if (mirror_ns->mirror_peer_uuids.count(m_remote_mirror_peer_uuid) == + 0) { + dout(15) << "skipping remote snapshot due to missing mirror peer" + << dendl; + continue; + } + + auto remote_snap_id = snap_info_it->first; + if (m_local_snap_id_start > 0 || m_local_snap_id_end != CEPH_NOSNAP) { + // we have a local mirror snapshot + if (m_local_mirror_snap_ns.is_non_primary()) { + // previously validated that it was linked to remote + ceph_assert(m_local_mirror_snap_ns.primary_mirror_uuid == + m_state_builder->remote_mirror_uuid); + + if (m_remote_snap_id_end == CEPH_NOSNAP) { + // haven't found the end snap so treat this as a candidate for unlink + unlink_snap_ids.insert(remote_snap_id); + } + if (m_local_mirror_snap_ns.complete && + m_local_mirror_snap_ns.primary_snap_id >= remote_snap_id) { + // skip past completed remote snapshot + m_remote_snap_id_start = remote_snap_id; + m_remote_mirror_snap_ns = *mirror_ns; + dout(15) << "skipping synced remote snapshot " << remote_snap_id + << dendl; + continue; + } else if (!m_local_mirror_snap_ns.complete && + m_local_mirror_snap_ns.primary_snap_id > remote_snap_id) { + // skip until we get to the in-progress remote snapshot + dout(15) << "skipping synced remote snapshot " << remote_snap_id + << " while search for in-progress sync" << dendl; + m_remote_snap_id_start = remote_snap_id; + m_remote_mirror_snap_ns = *mirror_ns; + continue; + } + } else if (m_local_mirror_snap_ns.state == + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED) { + // find the matching demotion snapshot in remote image + ceph_assert(m_local_snap_id_start > 0); + if (mirror_ns->state == + cls::rbd::MIRROR_SNAPSHOT_STATE_NON_PRIMARY_DEMOTED && + mirror_ns->primary_mirror_uuid == m_local_mirror_uuid && + mirror_ns->primary_snap_id == m_local_snap_id_start) { + dout(10) << "located matching demotion snapshot: " + << "remote_snap_id=" << remote_snap_id << ", " + << "local_snap_id=" << m_local_snap_id_start << dendl; + m_remote_snap_id_start = remote_snap_id; + split_brain = false; + continue; + } else if (m_remote_snap_id_start == 0) { + // still looking for our matching demotion snapshot + dout(15) << "skipping remote snapshot " << remote_snap_id << " " + << "while searching for demotion" << dendl; + split_brain = true; + continue; + } + } else { + // should not have been able to reach this + ceph_assert(false); + } + } else if (!mirror_ns->is_primary()) { + dout(15) << "skipping non-primary remote snapshot" << dendl; + continue; + } + + // found candidate snapshot to sync + ++m_pending_snapshots; + if (m_remote_snap_id_end != CEPH_NOSNAP) { + continue; + } + + // first primary snapshot where were are listed as a peer + m_remote_snap_id_end = remote_snap_id; + m_remote_mirror_snap_ns = *mirror_ns; + } + + if (m_remote_snap_id_start != 0 && + remote_image_ctx->snap_info.count(m_remote_snap_id_start) == 0) { + // the remote start snapshot was deleted out from under us + derr << "failed to locate remote start snapshot: " + << "snap_id=" << m_remote_snap_id_start << dendl; + split_brain = true; + } + + image_locker.unlock(); + + if (!split_brain) { + unlink_snap_ids.erase(m_remote_snap_id_start); + unlink_snap_ids.erase(m_remote_snap_id_end); + if (!unlink_snap_ids.empty()) { + locker->unlock(); + + // retry the unlinking process for a remote snapshot that we do not + // need anymore + auto remote_snap_id = *unlink_snap_ids.begin(); + dout(10) << "unlinking from remote snapshot " << remote_snap_id << dendl; + unlink_peer(remote_snap_id); + return; + } + + if (m_remote_snap_id_end != CEPH_NOSNAP) { + dout(10) << "found remote mirror snapshot: " + << "remote_snap_id_start=" << m_remote_snap_id_start << ", " + << "remote_snap_id_end=" << m_remote_snap_id_end << ", " + << "remote_snap_ns=" << m_remote_mirror_snap_ns << dendl; + if (m_remote_mirror_snap_ns.complete) { + locker->unlock(); + + if (m_local_snap_id_end != CEPH_NOSNAP && + !m_local_mirror_snap_ns.complete) { + // attempt to resume image-sync + dout(10) << "local image contains in-progress mirror snapshot" + << dendl; + get_local_image_state(); + } else { + copy_snapshots(); + } + return; + } else { + // might have raced with the creation of a remote mirror snapshot + // so we will need to refresh and rescan once it completes + dout(15) << "remote mirror snapshot not complete" << dendl; + } + } + } + + if (m_image_updated) { + // received update notification while scanning image, restart ... + m_image_updated = false; + locker->unlock(); + + dout(10) << "restarting snapshot scan due to remote update notification" + << dendl; + load_local_image_meta(); + return; + } + + if (is_replay_interrupted(locker)) { + return; + } else if (split_brain) { + derr << "split-brain detected: failed to find matching non-primary " + << "snapshot in remote image: " + << "local_snap_id_start=" << m_local_snap_id_start << ", " + << "local_snap_ns=" << m_local_mirror_snap_ns << dendl; + handle_replay_complete(locker, -EEXIST, "split-brain"); + return; + } else if (remote_demoted) { + dout(10) << "remote image demoted" << dendl; + handle_replay_complete(locker, -EREMOTEIO, "remote image demoted"); + return; + } + + dout(10) << "all remote snapshots synced: idling waiting for new snapshot" + << dendl; + ceph_assert(m_state == STATE_REPLAYING); + m_state = STATE_IDLE; + + notify_status_updated(); +} + +template <typename I> +void Replayer<I>::prune_non_primary_snapshot(uint64_t snap_id) { + dout(10) << "snap_id=" << snap_id << dendl; + + auto local_image_ctx = m_state_builder->local_image_ctx; + bool snap_valid = false; + cls::rbd::SnapshotNamespace snap_namespace; + std::string snap_name; + + { + std::shared_lock image_locker{local_image_ctx->image_lock}; + auto snap_info = local_image_ctx->get_snap_info(snap_id); + if (snap_info != nullptr) { + snap_valid = true; + snap_namespace = snap_info->snap_namespace; + snap_name = snap_info->name; + + ceph_assert(boost::get<cls::rbd::MirrorSnapshotNamespace>( + &snap_namespace) != nullptr); + } + } + + if (!snap_valid) { + load_local_image_meta(); + return; + } + + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_prune_non_primary_snapshot>(this); + local_image_ctx->operations->snap_remove(snap_namespace, snap_name, ctx); +} + +template <typename I> +void Replayer<I>::handle_prune_non_primary_snapshot(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "failed to prune non-primary snapshot: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to prune non-primary snapshot"); + return; + } + + if (is_replay_interrupted()) { + return; + } + + load_local_image_meta(); +} + +template <typename I> +void Replayer<I>::copy_snapshots() { + dout(10) << "remote_snap_id_start=" << m_remote_snap_id_start << ", " + << "remote_snap_id_end=" << m_remote_snap_id_end << ", " + << "local_snap_id_start=" << m_local_snap_id_start << dendl; + + ceph_assert(m_remote_snap_id_start != CEPH_NOSNAP); + ceph_assert(m_remote_snap_id_end > 0 && + m_remote_snap_id_end != CEPH_NOSNAP); + ceph_assert(m_local_snap_id_start != CEPH_NOSNAP); + + m_local_mirror_snap_ns = {}; + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_copy_snapshots>(this); + auto req = librbd::deep_copy::SnapshotCopyRequest<I>::create( + m_state_builder->remote_image_ctx, m_state_builder->local_image_ctx, + m_remote_snap_id_start, m_remote_snap_id_end, m_local_snap_id_start, + false, m_threads->work_queue, &m_local_mirror_snap_ns.snap_seqs, + ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_copy_snapshots(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to copy snapshots from remote to local image: " + << cpp_strerror(r) << dendl; + handle_replay_complete( + r, "failed to copy snapshots from remote to local image"); + return; + } + + dout(10) << "remote_snap_id_start=" << m_remote_snap_id_start << ", " + << "remote_snap_id_end=" << m_remote_snap_id_end << ", " + << "local_snap_id_start=" << m_local_snap_id_start << ", " + << "snap_seqs=" << m_local_mirror_snap_ns.snap_seqs << dendl; + get_remote_image_state(); +} + +template <typename I> +void Replayer<I>::get_remote_image_state() { + dout(10) << dendl; + + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_get_remote_image_state>(this); + auto req = librbd::mirror::snapshot::GetImageStateRequest<I>::create( + m_state_builder->remote_image_ctx, m_remote_snap_id_end, + &m_image_state, ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_get_remote_image_state(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to retrieve remote snapshot image state: " + << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to retrieve remote snapshot image state"); + return; + } + + create_non_primary_snapshot(); +} + +template <typename I> +void Replayer<I>::get_local_image_state() { + dout(10) << dendl; + + ceph_assert(m_local_snap_id_end != CEPH_NOSNAP); + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_get_local_image_state>(this); + auto req = librbd::mirror::snapshot::GetImageStateRequest<I>::create( + m_state_builder->local_image_ctx, m_local_snap_id_end, + &m_image_state, ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_get_local_image_state(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to retrieve local snapshot image state: " + << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to retrieve local snapshot image state"); + return; + } + + request_sync(); +} + +template <typename I> +void Replayer<I>::create_non_primary_snapshot() { + auto local_image_ctx = m_state_builder->local_image_ctx; + + if (m_local_snap_id_start > 0) { + std::shared_lock local_image_locker{local_image_ctx->image_lock}; + + auto local_snap_info_it = local_image_ctx->snap_info.find( + m_local_snap_id_start); + if (local_snap_info_it == local_image_ctx->snap_info.end()) { + local_image_locker.unlock(); + + derr << "failed to locate local snapshot " << m_local_snap_id_start + << dendl; + handle_replay_complete(-ENOENT, "failed to locate local start snapshot"); + return; + } + + auto mirror_ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &local_snap_info_it->second.snap_namespace); + ceph_assert(mirror_ns != nullptr); + + auto remote_image_ctx = m_state_builder->remote_image_ctx; + std::shared_lock remote_image_locker{remote_image_ctx->image_lock}; + + // (re)build a full mapping from remote to local snap ids for all user + // snapshots to support applying image state in the future + for (auto& [remote_snap_id, remote_snap_info] : + remote_image_ctx->snap_info) { + if (remote_snap_id >= m_remote_snap_id_end) { + break; + } + + // we can ignore all non-user snapshots since image state only includes + // user snapshots + if (boost::get<cls::rbd::UserSnapshotNamespace>( + &remote_snap_info.snap_namespace) == nullptr) { + continue; + } + + uint64_t local_snap_id = CEPH_NOSNAP; + if (mirror_ns->is_demoted() && !m_remote_mirror_snap_ns.is_demoted()) { + // if we are creating a non-primary snapshot following a demotion, + // re-build the full snapshot sequence since we don't have a valid + // snapshot mapping + auto local_snap_id_it = local_image_ctx->snap_ids.find( + {remote_snap_info.snap_namespace, remote_snap_info.name}); + if (local_snap_id_it != local_image_ctx->snap_ids.end()) { + local_snap_id = local_snap_id_it->second; + } + } else { + auto snap_seq_it = mirror_ns->snap_seqs.find(remote_snap_id); + if (snap_seq_it != mirror_ns->snap_seqs.end()) { + local_snap_id = snap_seq_it->second; + } + } + + if (m_local_mirror_snap_ns.snap_seqs.count(remote_snap_id) == 0 && + local_snap_id != CEPH_NOSNAP) { + dout(15) << "mapping remote snapshot " << remote_snap_id << " to " + << "local snapshot " << local_snap_id << dendl; + m_local_mirror_snap_ns.snap_seqs[remote_snap_id] = local_snap_id; + } + } + } + + dout(10) << "demoted=" << m_remote_mirror_snap_ns.is_demoted() << ", " + << "primary_mirror_uuid=" + << m_state_builder->remote_mirror_uuid << ", " + << "primary_snap_id=" << m_remote_snap_id_end << ", " + << "snap_seqs=" << m_local_mirror_snap_ns.snap_seqs << dendl; + + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_create_non_primary_snapshot>(this); + auto req = librbd::mirror::snapshot::CreateNonPrimaryRequest<I>::create( + local_image_ctx, m_remote_mirror_snap_ns.is_demoted(), + m_state_builder->remote_mirror_uuid, m_remote_snap_id_end, + m_local_mirror_snap_ns.snap_seqs, m_image_state, &m_local_snap_id_end, ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_create_non_primary_snapshot(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to create local mirror snapshot: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to create local mirror snapshot"); + return; + } + + dout(15) << "local_snap_id_end=" << m_local_snap_id_end << dendl; + + update_mirror_image_state(); +} + +template <typename I> +void Replayer<I>::update_mirror_image_state() { + if (m_local_snap_id_start > 0) { + request_sync(); + return; + } + + // a newly created non-primary image has a local mirror state of CREATING + // until this point so that we could avoid preserving the image until + // the first non-primary snapshot linked the two images together. + dout(10) << dendl; + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_update_mirror_image_state>(this); + auto req = librbd::mirror::ImageStateUpdateRequest<I>::create( + m_state_builder->local_image_ctx->md_ctx, + m_state_builder->local_image_ctx->id, + cls::rbd::MIRROR_IMAGE_STATE_ENABLED, {}, ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_update_mirror_image_state(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to update local mirror image state: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to update local mirror image state"); + return; + } + + request_sync(); +} + +template <typename I> +void Replayer<I>::request_sync() { + if (m_remote_mirror_snap_ns.clean_since_snap_id == m_remote_snap_id_start) { + dout(10) << "skipping unnecessary image copy: " + << "remote_snap_id_start=" << m_remote_snap_id_start << ", " + << "remote_mirror_snap_ns=" << m_remote_mirror_snap_ns << dendl; + apply_image_state(); + return; + } + + dout(10) << dendl; + std::unique_lock locker{m_lock}; + if (is_replay_interrupted(&locker)) { + return; + } + + auto ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_request_sync>(this)); + m_instance_watcher->notify_sync_request(m_state_builder->local_image_ctx->id, + ctx); +} + +template <typename I> +void Replayer<I>::handle_request_sync(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + if (is_replay_interrupted(&locker)) { + return; + } else if (r == -ECANCELED) { + dout(5) << "image-sync canceled" << dendl; + handle_replay_complete(&locker, r, "image-sync canceled"); + return; + } else if (r < 0) { + derr << "failed to request image-sync: " << cpp_strerror(r) << dendl; + handle_replay_complete(&locker, r, "failed to request image-sync"); + return; + } + + m_sync_in_progress = true; + locker.unlock(); + + copy_image(); +} + +template <typename I> +void Replayer<I>::copy_image() { + dout(10) << "remote_snap_id_start=" << m_remote_snap_id_start << ", " + << "remote_snap_id_end=" << m_remote_snap_id_end << ", " + << "local_snap_id_start=" << m_local_snap_id_start << ", " + << "last_copied_object_number=" + << m_local_mirror_snap_ns.last_copied_object_number << ", " + << "snap_seqs=" << m_local_mirror_snap_ns.snap_seqs << dendl; + + m_snapshot_bytes = 0; + m_snapshot_replay_start = ceph_clock_now(); + m_deep_copy_handler = new DeepCopyHandler(this); + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_copy_image>(this); + auto req = librbd::deep_copy::ImageCopyRequest<I>::create( + m_state_builder->remote_image_ctx, m_state_builder->local_image_ctx, + m_remote_snap_id_start, m_remote_snap_id_end, m_local_snap_id_start, false, + (m_local_mirror_snap_ns.last_copied_object_number > 0 ? + librbd::deep_copy::ObjectNumber{ + m_local_mirror_snap_ns.last_copied_object_number} : + librbd::deep_copy::ObjectNumber{}), + m_local_mirror_snap_ns.snap_seqs, m_deep_copy_handler, ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_copy_image(int r) { + dout(10) << "r=" << r << dendl; + + delete m_deep_copy_handler; + m_deep_copy_handler = nullptr; + + if (r < 0) { + derr << "failed to copy remote image to local image: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to copy remote image"); + return; + } + + { + std::unique_lock locker{m_lock}; + m_bytes_per_snapshot(m_snapshot_bytes); + auto time = ceph_clock_now() - m_snapshot_replay_start; + if (g_snapshot_perf_counters) { + g_snapshot_perf_counters->inc(l_rbd_mirror_snapshot_replay_bytes, + m_snapshot_bytes); + g_snapshot_perf_counters->inc(l_rbd_mirror_snapshot_replay_snapshots); + g_snapshot_perf_counters->tinc( + l_rbd_mirror_snapshot_replay_snapshots_time, time); + } + if (m_perf_counters) { + m_perf_counters->inc(l_rbd_mirror_snapshot_replay_bytes, m_snapshot_bytes); + m_perf_counters->inc(l_rbd_mirror_snapshot_replay_snapshots); + m_perf_counters->tinc(l_rbd_mirror_snapshot_replay_snapshots_time, time); + } + m_snapshot_bytes = 0; + } + + apply_image_state(); +} + +template <typename I> +void Replayer<I>::handle_copy_image_progress(uint64_t object_number, + uint64_t object_count) { + dout(10) << "object_number=" << object_number << ", " + << "object_count=" << object_count << dendl; + + std::unique_lock locker{m_lock}; + m_local_mirror_snap_ns.last_copied_object_number = std::min( + object_number, object_count); + m_local_object_count = object_count; + + update_non_primary_snapshot(false); +} + +template <typename I> +void Replayer<I>::handle_copy_image_read(uint64_t bytes_read) { + dout(20) << "bytes_read=" << bytes_read << dendl; + + std::unique_lock locker{m_lock}; + m_bytes_per_second(bytes_read); + m_snapshot_bytes += bytes_read; +} + +template <typename I> +void Replayer<I>::apply_image_state() { + dout(10) << dendl; + + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_apply_image_state>(this); + auto req = ApplyImageStateRequest<I>::create( + m_local_mirror_uuid, + m_state_builder->remote_mirror_uuid, + m_state_builder->local_image_ctx, + m_state_builder->remote_image_ctx, + m_image_state, ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_apply_image_state(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "failed to apply remote image state to local image: " + << cpp_strerror(r) << dendl; + handle_replay_complete(r, "failed to apply remote image state"); + return; + } + + std::unique_lock locker{m_lock}; + update_non_primary_snapshot(true); +} + +template <typename I> +void Replayer<I>::update_non_primary_snapshot(bool complete) { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + if (!complete) { + // disallow two in-flight updates if this isn't the completion of the sync + if (m_updating_sync_point) { + return; + } + m_updating_sync_point = true; + } else { + m_local_mirror_snap_ns.complete = true; + } + + dout(10) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_snapshot_set_copy_progress( + &op, m_local_snap_id_end, m_local_mirror_snap_ns.complete, + m_local_mirror_snap_ns.last_copied_object_number); + + auto ctx = new C_TrackedOp( + m_in_flight_op_tracker, new LambdaContext([this, complete](int r) { + handle_update_non_primary_snapshot(complete, r); + })); + auto aio_comp = create_rados_callback(ctx); + int r = m_state_builder->local_image_ctx->md_ctx.aio_operate( + m_state_builder->local_image_ctx->header_oid, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void Replayer<I>::handle_update_non_primary_snapshot(bool complete, int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to update local snapshot progress: " << cpp_strerror(r) + << dendl; + if (complete) { + // only fail if this was the final update + handle_replay_complete(r, "failed to update local snapshot progress"); + return; + } + } + + if (!complete) { + // periodic sync-point update -- do not advance state machine + std::unique_lock locker{m_lock}; + + ceph_assert(m_updating_sync_point); + m_updating_sync_point = false; + return; + } + + notify_image_update(); +} + +template <typename I> +void Replayer<I>::notify_image_update() { + dout(10) << dendl; + + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_notify_image_update>(this); + m_state_builder->local_image_ctx->notify_update(ctx); +} + +template <typename I> +void Replayer<I>::handle_notify_image_update(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to notify local image update: " << cpp_strerror(r) << dendl; + } + + unlink_peer(m_remote_snap_id_start); +} + +template <typename I> +void Replayer<I>::unlink_peer(uint64_t remote_snap_id) { + if (remote_snap_id == 0) { + finish_sync(); + return; + } + + // local snapshot fully synced -- we no longer depend on the sync + // start snapshot in the remote image + dout(10) << "remote_snap_id=" << remote_snap_id << dendl; + + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_unlink_peer>(this); + auto req = librbd::mirror::snapshot::UnlinkPeerRequest<I>::create( + m_state_builder->remote_image_ctx, remote_snap_id, + m_remote_mirror_peer_uuid, ctx); + req->send(); +} + +template <typename I> +void Replayer<I>::handle_unlink_peer(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "failed to unlink local peer from remote image: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to unlink local peer from remote image"); + return; + } + + finish_sync(); +} + +template <typename I> +void Replayer<I>::finish_sync() { + dout(10) << dendl; + + { + std::unique_lock locker{m_lock}; + notify_status_updated(); + + if (m_sync_in_progress) { + m_sync_in_progress = false; + m_instance_watcher->notify_sync_complete( + m_state_builder->local_image_ctx->id); + } + } + + if (is_replay_interrupted()) { + return; + } + + load_local_image_meta(); +} + +template <typename I> +void Replayer<I>::register_local_update_watcher() { + dout(10) << dendl; + + m_update_watch_ctx = new C_UpdateWatchCtx(this); + + int r = m_state_builder->local_image_ctx->state->register_update_watcher( + m_update_watch_ctx, &m_local_update_watcher_handle); + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_register_local_update_watcher>(this); + m_threads->work_queue->queue(ctx, r); +} + +template <typename I> +void Replayer<I>::handle_register_local_update_watcher(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to register local update watcher: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to register local image update watcher"); + m_state = STATE_COMPLETE; + + delete m_update_watch_ctx; + m_update_watch_ctx = nullptr; + + Context* on_init = nullptr; + std::swap(on_init, m_on_init_shutdown); + on_init->complete(r); + return; + } + + register_remote_update_watcher(); +} + +template <typename I> +void Replayer<I>::register_remote_update_watcher() { + dout(10) << dendl; + + int r = m_state_builder->remote_image_ctx->state->register_update_watcher( + m_update_watch_ctx, &m_remote_update_watcher_handle); + auto ctx = create_context_callback< + Replayer<I>, &Replayer<I>::handle_register_remote_update_watcher>(this); + m_threads->work_queue->queue(ctx, r); +} + +template <typename I> +void Replayer<I>::handle_register_remote_update_watcher(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to register remote update watcher: " << cpp_strerror(r) + << dendl; + handle_replay_complete(r, "failed to register remote image update watcher"); + m_state = STATE_COMPLETE; + + unregister_local_update_watcher(); + return; + } + + m_state = STATE_REPLAYING; + + Context* on_init = nullptr; + std::swap(on_init, m_on_init_shutdown); + on_init->complete(0); + + // delay initial snapshot scan until after we have alerted + // image replayer that we have initialized in case an error + // occurs + { + std::unique_lock locker{m_lock}; + notify_status_updated(); + } + + load_local_image_meta(); +} + +template <typename I> +void Replayer<I>::unregister_remote_update_watcher() { + dout(10) << dendl; + + auto ctx = create_context_callback< + Replayer<I>, + &Replayer<I>::handle_unregister_remote_update_watcher>(this); + m_state_builder->remote_image_ctx->state->unregister_update_watcher( + m_remote_update_watcher_handle, ctx); +} + +template <typename I> +void Replayer<I>::handle_unregister_remote_update_watcher(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to unregister remote update watcher: " << cpp_strerror(r) + << dendl; + } + + unregister_local_update_watcher(); +} + +template <typename I> +void Replayer<I>::unregister_local_update_watcher() { + dout(10) << dendl; + + auto ctx = create_context_callback< + Replayer<I>, + &Replayer<I>::handle_unregister_local_update_watcher>(this); + m_state_builder->local_image_ctx->state->unregister_update_watcher( + m_local_update_watcher_handle, ctx); +} + +template <typename I> +void Replayer<I>::handle_unregister_local_update_watcher(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to unregister local update watcher: " << cpp_strerror(r) + << dendl; + } + + delete m_update_watch_ctx; + m_update_watch_ctx = nullptr; + + wait_for_in_flight_ops(); +} + +template <typename I> +void Replayer<I>::wait_for_in_flight_ops() { + dout(10) << dendl; + + auto ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_wait_for_in_flight_ops>(this)); + m_in_flight_op_tracker.wait_for_ops(ctx); +} + +template <typename I> +void Replayer<I>::handle_wait_for_in_flight_ops(int r) { + dout(10) << "r=" << r << dendl; + + Context* on_shutdown = nullptr; + { + std::unique_lock locker{m_lock}; + ceph_assert(m_on_init_shutdown != nullptr); + std::swap(on_shutdown, m_on_init_shutdown); + } + on_shutdown->complete(m_error_code); +} + +template <typename I> +void Replayer<I>::handle_image_update_notify() { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + if (m_state == STATE_REPLAYING) { + dout(15) << "flagging snapshot rescan required" << dendl; + m_image_updated = true; + } else if (m_state == STATE_IDLE) { + m_state = STATE_REPLAYING; + locker.unlock(); + + dout(15) << "restarting idle replayer" << dendl; + load_local_image_meta(); + } +} + +template <typename I> +void Replayer<I>::handle_replay_complete(int r, + const std::string& description) { + std::unique_lock locker{m_lock}; + handle_replay_complete(&locker, r, description); +} + +template <typename I> +void Replayer<I>::handle_replay_complete(std::unique_lock<ceph::mutex>* locker, + int r, + const std::string& description) { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + if (m_sync_in_progress) { + m_sync_in_progress = false; + m_instance_watcher->notify_sync_complete( + m_state_builder->local_image_ctx->id); + } + + // don't set error code and description if resuming a pending + // shutdown + if (is_replay_interrupted(locker)) { + return; + } + + if (m_error_code == 0) { + m_error_code = r; + m_error_description = description; + } + + if (m_state != STATE_REPLAYING && m_state != STATE_IDLE) { + return; + } + + m_state = STATE_COMPLETE; + notify_status_updated(); +} + +template <typename I> +void Replayer<I>::notify_status_updated() { + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + dout(10) << dendl; + auto ctx = new C_TrackedOp(m_in_flight_op_tracker, new LambdaContext( + [this](int) { + m_replayer_listener->handle_notification(); + })); + m_threads->work_queue->queue(ctx, 0); +} + +template <typename I> +bool Replayer<I>::is_replay_interrupted() { + std::unique_lock locker{m_lock}; + return is_replay_interrupted(&locker); +} + +template <typename I> +bool Replayer<I>::is_replay_interrupted(std::unique_lock<ceph::mutex>* locker) { + if (m_state == STATE_COMPLETE) { + locker->unlock(); + + dout(10) << "resuming pending shut down" << dendl; + unregister_remote_update_watcher(); + return true; + } + return false; +} + +template <typename I> +void Replayer<I>::register_perf_counters() { + dout(5) << dendl; + + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + ceph_assert(m_perf_counters == nullptr); + + auto cct = static_cast<CephContext *>(m_state_builder->local_image_ctx->cct); + auto prio = cct->_conf.get_val<int64_t>("rbd_mirror_image_perf_stats_prio"); + PerfCountersBuilder plb(g_ceph_context, + "rbd_mirror_snapshot_image_" + m_image_spec, + l_rbd_mirror_snapshot_first, + l_rbd_mirror_snapshot_last); + plb.add_u64_counter(l_rbd_mirror_snapshot_replay_snapshots, + "snapshots", "Snapshots", "r", prio); + plb.add_time_avg(l_rbd_mirror_snapshot_replay_snapshots_time, + "snapshots_time", "Snapshots time", "rl", prio); + plb.add_u64_counter(l_rbd_mirror_snapshot_replay_bytes, "replay_bytes", + "Replayed data", "rb", prio, unit_t(UNIT_BYTES)); + m_perf_counters = plb.create_perf_counters(); + g_ceph_context->get_perfcounters_collection()->add(m_perf_counters); +} + +template <typename I> +void Replayer<I>::unregister_perf_counters() { + dout(5) << dendl; + ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); + + PerfCounters *perf_counters = nullptr; + std::swap(perf_counters, m_perf_counters); + + if (perf_counters != nullptr) { + g_ceph_context->get_perfcounters_collection()->remove(perf_counters); + delete perf_counters; + } +} + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::snapshot::Replayer<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h new file mode 100644 index 000000000..e3c4c2089 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h @@ -0,0 +1,346 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_REPLAYER_H +#define RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_REPLAYER_H + +#include "tools/rbd_mirror/image_replayer/Replayer.h" +#include "common/ceph_mutex.h" +#include "common/AsyncOpTracker.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/mirror/snapshot/Types.h" +#include "tools/rbd_mirror/image_replayer/TimeRollingMean.h" +#include <boost/accumulators/accumulators.hpp> +#include <boost/accumulators/statistics/stats.hpp> +#include <boost/accumulators/statistics/rolling_mean.hpp> +#include <string> +#include <type_traits> + +namespace librbd { + +struct ImageCtx; +namespace snapshot { template <typename I> class Replay; } + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <typename> struct InstanceWatcher; +class PoolMetaCache; +template <typename> struct Threads; + +namespace image_replayer { + +struct ReplayerListener; + +namespace snapshot { + +template <typename> class EventPreprocessor; +template <typename> class ReplayStatusFormatter; +template <typename> class StateBuilder; + +template <typename ImageCtxT> +class Replayer : public image_replayer::Replayer { +public: + static Replayer* create( + Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + StateBuilder<ImageCtxT>* state_builder, + ReplayerListener* replayer_listener) { + return new Replayer(threads, instance_watcher, local_mirror_uuid, + pool_meta_cache, state_builder, replayer_listener); + } + + Replayer( + Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + StateBuilder<ImageCtxT>* state_builder, + ReplayerListener* replayer_listener); + ~Replayer(); + + void destroy() override { + delete this; + } + + void init(Context* on_finish) override; + void shut_down(Context* on_finish) override; + + void flush(Context* on_finish) override; + + bool get_replay_status(std::string* description, Context* on_finish) override; + + bool is_replaying() const override { + std::unique_lock locker{m_lock}; + return (m_state == STATE_REPLAYING || m_state == STATE_IDLE); + } + + bool is_resync_requested() const override { + std::unique_lock locker{m_lock}; + return m_resync_requested; + } + + int get_error_code() const override { + std::unique_lock locker(m_lock); + return m_error_code; + } + + std::string get_error_description() const override { + std::unique_lock locker(m_lock); + return m_error_description; + } + + std::string get_image_spec() const { + std::unique_lock locker(m_lock); + return m_image_spec; + } + +private: + /** + * @verbatim + * + * <init> + * | + * v + * REGISTER_LOCAL_UPDATE_WATCHER + * | + * v + * REGISTER_REMOTE_UPDATE_WATCHER + * | + * v + * LOAD_LOCAL_IMAGE_META <----------------------------\ + * | | + * v (skip if not needed) | + * REFRESH_LOCAL_IMAGE | + * | | + * v (skip if not needed) | + * REFRESH_REMOTE_IMAGE | + * | | + * | (unused non-primary snapshot) | + * |\--------------> PRUNE_NON_PRIMARY_SNAPSHOT---/| + * | | + * | (interrupted sync) | + * |\--------------> GET_LOCAL_IMAGE_STATE ------\ | + * | | | + * | (new snapshot) | | + * |\--------------> COPY_SNAPSHOTS | | + * | | | | + * | v | | + * | GET_REMOTE_IMAGE_STATE | | + * | | | | + * | v | | + * | CREATE_NON_PRIMARY_SNAPSHOT | | + * | | | | + * | v (skip if not needed)| | + * | UPDATE_MIRROR_IMAGE_STATE | | + * | | | | + * | |/--------------------/ | + * | | | + * | v | + * | REQUEST_SYNC | + * | | | + * | v | + * | COPY_IMAGE | + * | | | + * | v | + * | APPLY_IMAGE_STATE | + * | | | + * | v | + * | UPDATE_NON_PRIMARY_SNAPSHOT | + * | | | + * | v | + * | NOTIFY_IMAGE_UPDATE | + * | | | + * | (interrupted unlink) v | + * |\--------------> UNLINK_PEER | + * | | | + * | v | + * | NOTIFY_LISTENER | + * | | | + * | \----------------------/| + * | | + * | (remote demoted) | + * \---------------> NOTIFY_LISTENER | + * | | | + * |/--------------------/ | + * | | + * | (update notification) | + * <idle> --------------------------------------------/ + * | + * v + * <shut down> + * | + * v + * UNREGISTER_REMOTE_UPDATE_WATCHER + * | + * v + * UNREGISTER_LOCAL_UPDATE_WATCHER + * | + * v + * WAIT_FOR_IN_FLIGHT_OPS + * | + * v + * <finish> + * + * @endverbatim + */ + + enum State { + STATE_INIT, + STATE_REPLAYING, + STATE_IDLE, + STATE_COMPLETE + }; + + struct C_UpdateWatchCtx; + struct DeepCopyHandler; + + Threads<ImageCtxT>* m_threads; + InstanceWatcher<ImageCtxT>* m_instance_watcher; + std::string m_local_mirror_uuid; + PoolMetaCache* m_pool_meta_cache; + StateBuilder<ImageCtxT>* m_state_builder; + ReplayerListener* m_replayer_listener; + + mutable ceph::mutex m_lock; + + State m_state = STATE_INIT; + + std::string m_image_spec; + Context* m_on_init_shutdown = nullptr; + + bool m_resync_requested = false; + int m_error_code = 0; + std::string m_error_description; + + C_UpdateWatchCtx* m_update_watch_ctx = nullptr; + uint64_t m_local_update_watcher_handle = 0; + uint64_t m_remote_update_watcher_handle = 0; + bool m_image_updated = false; + + AsyncOpTracker m_in_flight_op_tracker; + + uint64_t m_local_snap_id_start = 0; + uint64_t m_local_snap_id_end = CEPH_NOSNAP; + cls::rbd::MirrorSnapshotNamespace m_local_mirror_snap_ns; + uint64_t m_local_object_count = 0; + + std::string m_remote_mirror_peer_uuid; + uint64_t m_remote_snap_id_start = 0; + uint64_t m_remote_snap_id_end = CEPH_NOSNAP; + cls::rbd::MirrorSnapshotNamespace m_remote_mirror_snap_ns; + + librbd::mirror::snapshot::ImageState m_image_state; + DeepCopyHandler* m_deep_copy_handler = nullptr; + + TimeRollingMean m_bytes_per_second; + + uint64_t m_snapshot_bytes = 0; + boost::accumulators::accumulator_set< + uint64_t, boost::accumulators::stats< + boost::accumulators::tag::rolling_mean>> m_bytes_per_snapshot{ + boost::accumulators::tag::rolling_window::window_size = 2}; + utime_t m_snapshot_replay_start; + + uint32_t m_pending_snapshots = 0; + + bool m_remote_image_updated = false; + bool m_updating_sync_point = false; + bool m_sync_in_progress = false; + + PerfCounters *m_perf_counters = nullptr; + + void load_local_image_meta(); + void handle_load_local_image_meta(int r); + + void refresh_local_image(); + void handle_refresh_local_image(int r); + + void refresh_remote_image(); + void handle_refresh_remote_image(int r); + + void scan_local_mirror_snapshots(std::unique_lock<ceph::mutex>* locker); + void scan_remote_mirror_snapshots(std::unique_lock<ceph::mutex>* locker); + + void prune_non_primary_snapshot(uint64_t snap_id); + void handle_prune_non_primary_snapshot(int r); + + void copy_snapshots(); + void handle_copy_snapshots(int r); + + void get_remote_image_state(); + void handle_get_remote_image_state(int r); + + void get_local_image_state(); + void handle_get_local_image_state(int r); + + void create_non_primary_snapshot(); + void handle_create_non_primary_snapshot(int r); + + void update_mirror_image_state(); + void handle_update_mirror_image_state(int r); + + void request_sync(); + void handle_request_sync(int r); + + void copy_image(); + void handle_copy_image(int r); + void handle_copy_image_progress(uint64_t object_number, + uint64_t object_count); + void handle_copy_image_read(uint64_t bytes_read); + + void apply_image_state(); + void handle_apply_image_state(int r); + + void update_non_primary_snapshot(bool complete); + void handle_update_non_primary_snapshot(bool complete, int r); + + void notify_image_update(); + void handle_notify_image_update(int r); + + void unlink_peer(uint64_t remote_snap_id); + void handle_unlink_peer(int r); + + void finish_sync(); + + void register_local_update_watcher(); + void handle_register_local_update_watcher(int r); + + void register_remote_update_watcher(); + void handle_register_remote_update_watcher(int r); + + void unregister_remote_update_watcher(); + void handle_unregister_remote_update_watcher(int r); + + void unregister_local_update_watcher(); + void handle_unregister_local_update_watcher(int r); + + void wait_for_in_flight_ops(); + void handle_wait_for_in_flight_ops(int r); + + void handle_image_update_notify(); + + void handle_replay_complete(int r, const std::string& description); + void handle_replay_complete(std::unique_lock<ceph::mutex>* locker, + int r, const std::string& description); + void notify_status_updated(); + + bool is_replay_interrupted(); + bool is_replay_interrupted(std::unique_lock<ceph::mutex>* lock); + + void register_perf_counters(); + void unregister_perf_counters(); +}; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::snapshot::Replayer<librbd::ImageCtx>; + +#endif // RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_REPLAYER_H diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc new file mode 100644 index 000000000..ca3e6918b --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc @@ -0,0 +1,120 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "StateBuilder.h" +#include "include/ceph_assert.h" +#include "include/Context.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ImageCtx.h" +#include "librbd/mirror/snapshot/ImageMeta.h" +#include "tools/rbd_mirror/image_replayer/snapshot/CreateLocalImageRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/PrepareReplayRequest.h" +#include "tools/rbd_mirror/image_replayer/snapshot/Replayer.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::snapshot::" \ + << "StateBuilder: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +template <typename I> +StateBuilder<I>::StateBuilder(const std::string& global_image_id) + : image_replayer::StateBuilder<I>(global_image_id) { +} + +template <typename I> +StateBuilder<I>::~StateBuilder() { + ceph_assert(local_image_meta == nullptr); +} + +template <typename I> +void StateBuilder<I>::close(Context* on_finish) { + dout(10) << dendl; + + delete local_image_meta; + local_image_meta = nullptr; + + // close the remote image after closing the local + // image in case the remote cluster is unreachable and + // we cannot close it. + on_finish = new LambdaContext([this, on_finish](int) { + this->close_remote_image(on_finish); + }); + this->close_local_image(on_finish); +} + +template <typename I> +bool StateBuilder<I>::is_disconnected() const { + return false; +} + +template <typename I> +bool StateBuilder<I>::is_linked_impl() const { + // the remote has to have us registered as a peer + return !remote_mirror_peer_uuid.empty(); +} + +template <typename I> +cls::rbd::MirrorImageMode StateBuilder<I>::get_mirror_image_mode() const { + return cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT; +} + +template <typename I> +image_sync::SyncPointHandler* StateBuilder<I>::create_sync_point_handler() { + dout(10) << dendl; + + // TODO + ceph_assert(false); + return nullptr; +} + +template <typename I> +BaseRequest* StateBuilder<I>::create_local_image_request( + Threads<I>* threads, + librados::IoCtx& local_io_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + Context* on_finish) { + return CreateLocalImageRequest<I>::create( + threads, local_io_ctx, this->remote_image_ctx, global_image_id, + pool_meta_cache, progress_ctx, this, on_finish); +} + +template <typename I> +BaseRequest* StateBuilder<I>::create_prepare_replay_request( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + bool* resync_requested, + bool* syncing, + Context* on_finish) { + return PrepareReplayRequest<I>::create( + local_mirror_uuid, progress_ctx, this, resync_requested, syncing, + on_finish); +} + +template <typename I> +image_replayer::Replayer* StateBuilder<I>::create_replayer( + Threads<I>* threads, + InstanceWatcher<I>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + ReplayerListener* replayer_listener) { + return Replayer<I>::create( + threads, instance_watcher, local_mirror_uuid, pool_meta_cache, this, + replayer_listener); +} + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_replayer::snapshot::StateBuilder<librbd::ImageCtx>; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h new file mode 100644 index 000000000..a4ab82982 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h @@ -0,0 +1,93 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_STATE_BUILDER_H +#define CEPH_RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_STATE_BUILDER_H + +#include "tools/rbd_mirror/image_replayer/StateBuilder.h" +#include <string> + +struct Context; + +namespace librbd { + +struct ImageCtx; + +namespace mirror { +namespace snapshot { + +template <typename> class ImageMeta; + +} // namespace snapshot +} // namespace mirror +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { + +template <typename> class SyncPointHandler; + +template <typename ImageCtxT> +class StateBuilder : public image_replayer::StateBuilder<ImageCtxT> { +public: + static StateBuilder* create(const std::string& global_image_id) { + return new StateBuilder(global_image_id); + } + + StateBuilder(const std::string& global_image_id); + ~StateBuilder() override; + + void close(Context* on_finish) override; + + bool is_disconnected() const override; + + cls::rbd::MirrorImageMode get_mirror_image_mode() const override; + + image_sync::SyncPointHandler* create_sync_point_handler() override; + + bool replay_requires_remote_image() const override { + return true; + } + + BaseRequest* create_local_image_request( + Threads<ImageCtxT>* threads, + librados::IoCtx& local_io_ctx, + const std::string& global_image_id, + PoolMetaCache* pool_meta_cache, + ProgressContext* progress_ctx, + Context* on_finish) override; + + BaseRequest* create_prepare_replay_request( + const std::string& local_mirror_uuid, + ProgressContext* progress_ctx, + bool* resync_requested, + bool* syncing, + Context* on_finish) override; + + image_replayer::Replayer* create_replayer( + Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, + const std::string& local_mirror_uuid, + PoolMetaCache* pool_meta_cache, + ReplayerListener* replayer_listener) override; + + SyncPointHandler<ImageCtxT>* sync_point_handler = nullptr; + + std::string remote_mirror_peer_uuid; + + librbd::mirror::snapshot::ImageMeta<ImageCtxT>* local_image_meta = nullptr; + +private: + bool is_linked_impl() const override; +}; + +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_replayer::snapshot::StateBuilder<librbd::ImageCtx>; + +#endif // CEPH_RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_STATE_BUILDER_H diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Utils.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Utils.cc new file mode 100644 index 000000000..7c20410cb --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Utils.cc @@ -0,0 +1,65 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "Utils.h" +#include "common/debug.h" +#include "common/errno.h" +#include "cls/rbd/cls_rbd_types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_replayer::snapshot::util::" \ + << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { +namespace util { + +uint64_t compute_remote_snap_id( + const ceph::shared_mutex& local_image_lock, + const std::map<librados::snap_t, librbd::SnapInfo>& local_snap_infos, + uint64_t local_snap_id, const std::string& remote_mirror_uuid) { + ceph_assert(ceph_mutex_is_locked(local_image_lock)); + + // Search our local non-primary snapshots for a mapping to the remote + // snapshot. The non-primary mirror snapshot with the mappings will always + // come at or after the snapshot we are searching against + for (auto snap_it = local_snap_infos.lower_bound(local_snap_id); + snap_it != local_snap_infos.end(); ++snap_it) { + auto mirror_ns = boost::get<cls::rbd::MirrorSnapshotNamespace>( + &snap_it->second.snap_namespace); + if (mirror_ns == nullptr || !mirror_ns->is_non_primary()) { + continue; + } + + if (mirror_ns->primary_mirror_uuid != remote_mirror_uuid) { + dout(20) << "local snapshot " << snap_it->first << " not tied to remote" + << dendl; + continue; + } else if (local_snap_id == snap_it->first) { + dout(15) << "local snapshot " << local_snap_id << " maps to " + << "remote snapshot " << mirror_ns->primary_snap_id << dendl; + return mirror_ns->primary_snap_id; + } + + const auto& snap_seqs = mirror_ns->snap_seqs; + for (auto [remote_snap_id_seq, local_snap_id_seq] : snap_seqs) { + if (local_snap_id_seq == local_snap_id) { + dout(15) << "local snapshot " << local_snap_id << " maps to " + << "remote snapshot " << remote_snap_id_seq << dendl; + return remote_snap_id_seq; + } + } + } + + return CEPH_NOSNAP; +} + +} // namespace util +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Utils.h b/src/tools/rbd_mirror/image_replayer/snapshot/Utils.h new file mode 100644 index 000000000..8efc58685 --- /dev/null +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Utils.h @@ -0,0 +1,30 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_UTILS_H +#define RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_UTILS_H + +#include "include/int_types.h" +#include "include/rados/librados.hpp" +#include "common/ceph_mutex.h" +#include "librbd/Types.h" +#include <map> + +namespace rbd { +namespace mirror { +namespace image_replayer { +namespace snapshot { +namespace util { + +uint64_t compute_remote_snap_id( + const ceph::shared_mutex& local_image_lock, + const std::map<librados::snap_t, librbd::SnapInfo>& local_snap_infos, + uint64_t local_snap_id, const std::string& remote_mirror_uuid); + +} // namespace util +} // namespace snapshot +} // namespace image_replayer +} // namespace mirror +} // namespace rbd + +#endif // RBD_MIRROR_IMAGE_REPLAYER_SNAPSHOT_UTILS_H |