diff options
Diffstat (limited to '')
| -rw-r--r-- | src/tools/rbd_mirror/RemotePoolPoller.cc | 267 |
1 files changed, 267 insertions, 0 deletions
diff --git a/src/tools/rbd_mirror/RemotePoolPoller.cc b/src/tools/rbd_mirror/RemotePoolPoller.cc new file mode 100644 index 000000000..8bfb35d4a --- /dev/null +++ b/src/tools/rbd_mirror/RemotePoolPoller.cc @@ -0,0 +1,267 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "RemotePoolPoller.h" +#include "include/ceph_assert.h" +#include "common/debug.h" +#include "common/errno.h" +#include "common/Timer.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "librbd/asio/ContextWQ.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::RemotePoolPoller: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { + +static const double POLL_INTERVAL_SECONDS = 30; + +using librbd::util::create_rados_callback; + +template <typename I> +RemotePoolPoller<I>::~RemotePoolPoller() { + ceph_assert(m_timer_task == nullptr); +} + +template <typename I> +void RemotePoolPoller<I>::init(Context* on_finish) { + dout(10) << dendl; + + ceph_assert(m_state == STATE_INITIALIZING); + ceph_assert(m_on_finish == nullptr); + m_on_finish = on_finish; + + get_mirror_uuid(); +} + +template <typename I> +void RemotePoolPoller<I>::shut_down(Context* on_finish) { + dout(10) << dendl; + + std::unique_lock locker(m_threads->timer_lock); + ceph_assert(m_state == STATE_POLLING); + m_state = STATE_SHUTTING_DOWN; + + if (m_timer_task == nullptr) { + // currently executing a poll + ceph_assert(m_on_finish == nullptr); + m_on_finish = on_finish; + return; + } + + m_threads->timer->cancel_event(m_timer_task); + m_timer_task = nullptr; + m_threads->work_queue->queue(on_finish, 0); +} + +template <typename I> +void RemotePoolPoller<I>::get_mirror_uuid() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_uuid_get_start(&op); + + auto aio_comp = create_rados_callback< + RemotePoolPoller<I>, &RemotePoolPoller<I>::handle_get_mirror_uuid>(this); + m_out_bl.clear(); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void RemotePoolPoller<I>::handle_get_mirror_uuid(int r) { + dout(10) << "r=" << r << dendl; + std::string remote_mirror_uuid; + if (r >= 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_uuid_get_finish(&it, &remote_mirror_uuid); + if (r >= 0 && remote_mirror_uuid.empty()) { + r = -ENOENT; + } + } + + if (r < 0) { + if (r == -ENOENT) { + dout(5) << "remote mirror uuid missing" << dendl; + } else { + derr << "failed to retrieve remote mirror uuid: " << cpp_strerror(r) + << dendl; + } + + m_remote_pool_meta.mirror_uuid = ""; + } + + // if we have the mirror uuid, we will poll until shut down + if (m_state == STATE_INITIALIZING) { + if (r < 0) { + schedule_task(r); + return; + } + + m_state = STATE_POLLING; + } + + dout(10) << "remote_mirror_uuid=" << remote_mirror_uuid << dendl; + if (m_remote_pool_meta.mirror_uuid != remote_mirror_uuid) { + m_remote_pool_meta.mirror_uuid = remote_mirror_uuid; + m_updated = true; + } + + mirror_peer_ping(); +} + +template <typename I> +void RemotePoolPoller<I>::mirror_peer_ping() { + dout(10) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_peer_ping(&op, m_site_name, m_local_mirror_uuid); + + auto aio_comp = create_rados_callback< + RemotePoolPoller<I>, &RemotePoolPoller<I>::handle_mirror_peer_ping>(this); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void RemotePoolPoller<I>::handle_mirror_peer_ping(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -EOPNOTSUPP) { + // older OSD that doesn't support snaphot-based mirroring, so no need + // to query remote peers + dout(10) << "remote peer does not support snapshot-based mirroring" + << dendl; + notify_listener(); + return; + } else if (r < 0) { + // we can still see if we can perform a peer list and find outselves + derr << "failed to ping remote mirror peer: " << cpp_strerror(r) << dendl; + } + + mirror_peer_list(); +} + +template <typename I> +void RemotePoolPoller<I>::mirror_peer_list() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_peer_list_start(&op); + + auto aio_comp = create_rados_callback< + RemotePoolPoller<I>, &RemotePoolPoller<I>::handle_mirror_peer_list>(this); + m_out_bl.clear(); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template <typename I> +void RemotePoolPoller<I>::handle_mirror_peer_list(int r) { + dout(10) << "r=" << r << dendl; + + std::vector<cls::rbd::MirrorPeer> peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + derr << "failed to retrieve mirror peers: " << cpp_strerror(r) << dendl; + } + + cls::rbd::MirrorPeer* matched_peer = nullptr; + for (auto& peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_RX) { + continue; + } + + if (peer.mirror_uuid == m_local_mirror_uuid) { + matched_peer = &peer; + break; + } else if (peer.site_name == m_site_name) { + // keep searching in case we hit an exact match by fsid + matched_peer = &peer; + } + } + + // older OSDs don't support peer ping so we might fail to find a match, + // which will prevent snapshot mirroring from functioning + std::string remote_mirror_peer_uuid; + if (matched_peer != nullptr) { + remote_mirror_peer_uuid = matched_peer->uuid; + } + + dout(10) << "remote_mirror_peer_uuid=" << remote_mirror_peer_uuid << dendl; + if (m_remote_pool_meta.mirror_peer_uuid != remote_mirror_peer_uuid) { + m_remote_pool_meta.mirror_peer_uuid = remote_mirror_peer_uuid; + m_updated = true; + } + + notify_listener(); +} + +template <typename I> +void RemotePoolPoller<I>::notify_listener() { + bool updated = false; + std::swap(updated, m_updated); + if (updated) { + dout(10) << dendl; + m_listener.handle_updated(m_remote_pool_meta); + } + + schedule_task(0); +} + +template <typename I> +void RemotePoolPoller<I>::schedule_task(int r) { + std::unique_lock locker{m_threads->timer_lock}; + + if (m_state == STATE_POLLING) { + dout(10) << dendl; + + ceph_assert(m_timer_task == nullptr); + m_timer_task = new LambdaContext([this](int) { + handle_task(); + }); + + m_threads->timer->add_event_after(POLL_INTERVAL_SECONDS, m_timer_task); + } + + // finish init or shut down callback + if (m_on_finish != nullptr) { + locker.unlock(); + Context* on_finish = nullptr; + std::swap(on_finish, m_on_finish); + on_finish->complete(m_state == STATE_SHUTTING_DOWN ? 0 : r); + } +} + +template <typename I> +void RemotePoolPoller<I>::handle_task() { + dout(10) << dendl; + + ceph_assert(ceph_mutex_is_locked_by_me(m_threads->timer_lock)); + m_timer_task = nullptr; + + auto ctx = new LambdaContext([this](int) { + get_mirror_uuid(); + }); + m_threads->work_queue->queue(ctx); +} + +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::RemotePoolPoller<librbd::ImageCtx>; |