7 files changed, 683 insertions, 0 deletions
diff --git a/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc
new file mode 100644
index 000000000..1bd5d77f0
--- /dev/null
+++ b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.cc
@@ -0,0 +1,172 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "SyncPointCreateRequest.h"
+#include "include/uuid.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+#include "tools/rbd_mirror/image_sync/Types.h"
+#include "tools/rbd_mirror/image_sync/Utils.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_sync::SyncPointCreateRequest: " \
+                           << this << " " << __func__
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+SyncPointCreateRequest<I>::SyncPointCreateRequest(
+    I *remote_image_ctx,
+    const std::string &local_mirror_uuid,
+    SyncPointHandler* sync_point_handler,
+    Context *on_finish)
+  : m_remote_image_ctx(remote_image_ctx),
+    m_local_mirror_uuid(local_mirror_uuid),
+    m_sync_point_handler(sync_point_handler),
+    m_on_finish(on_finish) {
+  m_sync_points_copy = m_sync_point_handler->get_sync_points();
+  ceph_assert(m_sync_points_copy.size() < 2);
+
+  // initialize the updated client meta with the new sync point
+  m_sync_points_copy.emplace_back();
+  if (m_sync_points_copy.size() > 1) {
+    m_sync_points_copy.back().from_snap_name =
+      m_sync_points_copy.front().snap_name;
+  }
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send() {
+  send_update_sync_points();
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send_update_sync_points() {
+  uuid_d uuid_gen;
+  uuid_gen.generate_random();
+
+  auto& sync_point = m_sync_points_copy.back();
+  sync_point.snap_name = util::get_snapshot_name_prefix(m_local_mirror_uuid) +
+                         uuid_gen.to_string();
+
+  auto ctx = create_context_callback<
+    SyncPointCreateRequest<I>,
+    &SyncPointCreateRequest<I>::handle_update_sync_points>(this);
+  m_sync_point_handler->update_sync_points(
+    m_sync_point_handler->get_snap_seqs(), m_sync_points_copy, false, ctx);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::handle_update_sync_points(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    derr << ": failed to update client data: " << cpp_strerror(r)
+         << dendl;
+    finish(r);
+    return;
+  }
+
+  send_refresh_image();
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send_refresh_image() {
+  dout(20) << dendl;
+
+  Context *ctx = create_context_callback<
+    SyncPointCreateRequest<I>, &SyncPointCreateRequest<I>::handle_refresh_image>(
+      this);
+  m_remote_image_ctx->state->refresh(ctx);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::handle_refresh_image(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    derr << ": remote image refresh failed: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_create_snap();
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send_create_snap() {
+  dout(20) << dendl;
+
+  auto& sync_point = m_sync_points_copy.back();
+
+  Context *ctx = create_context_callback<
+    SyncPointCreateRequest<I>, &SyncPointCreateRequest<I>::handle_create_snap>(
+      this);
+  m_remote_image_ctx->operations->snap_create(
+    cls::rbd::UserSnapshotNamespace(), sync_point.snap_name.c_str(),
+    librbd::SNAP_CREATE_FLAG_SKIP_NOTIFY_QUIESCE, m_prog_ctx, ctx);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::handle_create_snap(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  if (r == -EEXIST) {
+    send_update_sync_points();
+    return;
+  } else if (r < 0) {
+    derr << ": failed to create snapshot: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_final_refresh_image();
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::send_final_refresh_image() {
+  dout(20) << dendl;
+
+  Context *ctx = create_context_callback<
+    SyncPointCreateRequest<I>,
+    &SyncPointCreateRequest<I>::handle_final_refresh_image>(this);
+  m_remote_image_ctx->state->refresh(ctx);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::handle_final_refresh_image(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    derr << ": failed to refresh image for snapshot: " << cpp_strerror(r)
+         << dendl;
+    finish(r);
+    return;
+  }
+
+  finish(0);
+}
+
+template <typename I>
+void SyncPointCreateRequest<I>::finish(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_sync::SyncPointCreateRequest<librbd::ImageCtx>;
diff --git a/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h
new file mode 100644
index 000000000..9b52b8374
--- /dev/null
+++ b/src/tools/rbd_mirror/image_sync/SyncPointCreateRequest.h
@@ -0,0 +1,93 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H
+#define RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H
+
+#include "librbd/internal.h"
+#include "Types.h"
+#include <string>
+
+class Context;
+namespace journal { class Journaler; }
+namespace librbd { class ImageCtx; }
+namespace librbd { namespace journal { struct MirrorPeerClientMeta; } }
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class SyncPointCreateRequest {
+public:
+  static SyncPointCreateRequest* create(
+      ImageCtxT *remote_image_ctx,
+      const std::string &local_mirror_uuid,
+      SyncPointHandler* sync_point_handler,
+      Context *on_finish) {
+    return new SyncPointCreateRequest(remote_image_ctx, local_mirror_uuid,
+                                      sync_point_handler, on_finish);
+  }
+
+  SyncPointCreateRequest(
+      ImageCtxT *remote_image_ctx,
+      const std::string &local_mirror_uuid,
+      SyncPointHandler* sync_point_handler,
+      Context *on_finish);
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * UPDATE_SYNC_POINTS < . .
+   *    |                   .
+   *    v                   .
+   * REFRESH_IMAGE          .
+   *    |                   . (repeat on EEXIST)
+   *    v                   .
+   * CREATE_SNAP  . . . . . .
+   *    |
+   *    v
+   * REFRESH_IMAGE
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  ImageCtxT *m_remote_image_ctx;
+  std::string m_local_mirror_uuid;
+  SyncPointHandler* m_sync_point_handler;
+  Context *m_on_finish;
+
+  SyncPoints m_sync_points_copy;
+  librbd::NoOpProgressContext m_prog_ctx;
+
+  void send_update_sync_points();
+  void handle_update_sync_points(int r);
+
+  void send_refresh_image();
+  void handle_refresh_image(int r);
+
+  void send_create_snap();
+  void handle_create_snap(int r);
+
+  void send_final_refresh_image();
+  void handle_final_refresh_image(int r);
+
+  void finish(int r);
+};
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_sync::SyncPointCreateRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_CREATE_REQUEST_H
diff --git a/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc
new file mode 100644
index 000000000..d1cd32b39
--- /dev/null
+++ b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.cc
@@ -0,0 +1,213 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "SyncPointPruneRequest.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+#include <set>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_sync::SyncPointPruneRequest: " \
+                           << this << " " << __func__
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+SyncPointPruneRequest<I>::SyncPointPruneRequest(
+    I *remote_image_ctx,
+    bool sync_complete,
+    SyncPointHandler* sync_point_handler,
+    Context *on_finish)
+  : m_remote_image_ctx(remote_image_ctx),
+    m_sync_complete(sync_complete),
+    m_sync_point_handler(sync_point_handler),
+    m_on_finish(on_finish) {
+  m_sync_points_copy = m_sync_point_handler->get_sync_points();
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send() {
+  if (m_sync_points_copy.empty()) {
+    send_remove_snap();
+    return;
+  }
+
+  if (m_sync_complete) {
+    // if sync is complete, we can remove the master sync point
+    auto it = m_sync_points_copy.begin();
+    auto& sync_point = *it;
+
+    ++it;
+    if (it == m_sync_points_copy.end() ||
+        it->from_snap_name != sync_point.snap_name) {
+      m_snap_names.push_back(sync_point.snap_name);
+    }
+
+    if (!sync_point.from_snap_name.empty()) {
+      m_snap_names.push_back(sync_point.from_snap_name);
+    }
+  } else {
+    // if we have more than one sync point or invalid sync points,
+    // trim them off
+    std::shared_lock image_locker{m_remote_image_ctx->image_lock};
+    std::set<std::string> snap_names;
+    for (auto it = m_sync_points_copy.rbegin();
+         it != m_sync_points_copy.rend(); ++it) {
+      auto& sync_point = *it;
+      if (&sync_point == &m_sync_points_copy.front()) {
+        if (m_remote_image_ctx->get_snap_id(
+	      cls::rbd::UserSnapshotNamespace(), sync_point.snap_name) ==
+              CEPH_NOSNAP) {
+          derr << ": failed to locate sync point snapshot: "
+               << sync_point.snap_name << dendl;
+        } else if (!sync_point.from_snap_name.empty()) {
+          derr << ": unexpected from_snap_name in primary sync point: "
+               << sync_point.from_snap_name << dendl;
+        } else {
+          // first sync point is OK -- keep it
+          break;
+        }
+        m_invalid_master_sync_point = true;
+      }
+
+      if (snap_names.count(sync_point.snap_name) == 0) {
+        snap_names.insert(sync_point.snap_name);
+        m_snap_names.push_back(sync_point.snap_name);
+      }
+
+      auto& front_sync_point = m_sync_points_copy.front();
+      if (!sync_point.from_snap_name.empty() &&
+          snap_names.count(sync_point.from_snap_name) == 0 &&
+          sync_point.from_snap_name != front_sync_point.snap_name) {
+        snap_names.insert(sync_point.from_snap_name);
+        m_snap_names.push_back(sync_point.from_snap_name);
+      }
+    }
+  }
+
+  send_remove_snap();
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send_remove_snap() {
+  if (m_snap_names.empty()) {
+    send_refresh_image();
+    return;
+  }
+
+  const std::string &snap_name = m_snap_names.front();
+
+  dout(20) << ": snap_name=" << snap_name << dendl;
+
+  Context *ctx = create_context_callback<
+    SyncPointPruneRequest<I>, &SyncPointPruneRequest<I>::handle_remove_snap>(
+      this);
+  m_remote_image_ctx->operations->snap_remove(cls::rbd::UserSnapshotNamespace(),
+					      snap_name.c_str(),
+					      ctx);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::handle_remove_snap(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  ceph_assert(!m_snap_names.empty());
+  std::string snap_name = m_snap_names.front();
+  m_snap_names.pop_front();
+
+  if (r == -ENOENT) {
+    r = 0;
+  }
+  if (r < 0) {
+    derr << ": failed to remove snapshot '" << snap_name << "': "
+         << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_remove_snap();
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send_refresh_image() {
+  dout(20) << dendl;
+
+  Context *ctx = create_context_callback<
+    SyncPointPruneRequest<I>, &SyncPointPruneRequest<I>::handle_refresh_image>(
+      this);
+  m_remote_image_ctx->state->refresh(ctx);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::handle_refresh_image(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    derr << ": remote image refresh failed: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  send_update_sync_points();
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::send_update_sync_points() {
+  dout(20) << dendl;
+
+  if (m_sync_complete) {
+    m_sync_points_copy.pop_front();
+  } else {
+    while (m_sync_points_copy.size() > 1) {
+      m_sync_points_copy.pop_back();
+    }
+    if (m_invalid_master_sync_point) {
+      // all subsequent sync points would have been pruned
+      m_sync_points_copy.clear();
+    }
+  }
+
+  auto ctx = create_context_callback<
+    SyncPointPruneRequest<I>,
+    &SyncPointPruneRequest<I>::handle_update_sync_points>(this);
+  m_sync_point_handler->update_sync_points(
+    m_sync_point_handler->get_snap_seqs(), m_sync_points_copy,
+    m_sync_complete, ctx);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::handle_update_sync_points(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  if (r < 0) {
+    derr << ": failed to update client data: " << cpp_strerror(r)
+         << dendl;
+    finish(r);
+    return;
+  }
+
+  finish(0);
+}
+
+template <typename I>
+void SyncPointPruneRequest<I>::finish(int r) {
+  dout(20) << ": r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_sync::SyncPointPruneRequest<librbd::ImageCtx>;
diff --git a/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h
new file mode 100644
index 000000000..08bf840b1
--- /dev/null
+++ b/src/tools/rbd_mirror/image_sync/SyncPointPruneRequest.h
@@ -0,0 +1,91 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H
+#define RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H
+
+#include "tools/rbd_mirror/image_sync/Types.h"
+#include <list>
+#include <string>
+
+class Context;
+namespace journal { class Journaler; }
+namespace librbd { class ImageCtx; }
+namespace librbd { namespace journal { struct MirrorPeerClientMeta; } }
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class SyncPointPruneRequest {
+public:
+  static SyncPointPruneRequest* create(
+      ImageCtxT *remote_image_ctx,
+      bool sync_complete,
+      SyncPointHandler* sync_point_handler,
+      Context *on_finish) {
+    return new SyncPointPruneRequest(remote_image_ctx, sync_complete,
+                                     sync_point_handler, on_finish);
+  }
+
+  SyncPointPruneRequest(
+      ImageCtxT *remote_image_ctx,
+      bool sync_complete,
+      SyncPointHandler* sync_point_handler,
+      Context *on_finish);
+
+  void send();
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    |    . . . . .
+   *    |    .       .
+   *    v    v       . (repeat if from snap
+   * REMOVE_SNAP . . .  unused by other sync)
+   *    |
+   *    v
+   * REFRESH_IMAGE
+   *    |
+   *    v
+   * UPDATE_CLIENT
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  ImageCtxT *m_remote_image_ctx;
+  bool m_sync_complete;
+  SyncPointHandler* m_sync_point_handler;
+  Context *m_on_finish;
+
+  SyncPoints m_sync_points_copy;
+  std::list<std::string> m_snap_names;
+
+  bool m_invalid_master_sync_point = false;
+
+  void send_remove_snap();
+  void handle_remove_snap(int r);
+
+  void send_refresh_image();
+  void handle_refresh_image(int r);
+
+  void send_update_sync_points();
+  void handle_update_sync_points(int r);
+
+  void finish(int r);
+};
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_sync::SyncPointPruneRequest<librbd::ImageCtx>;
+
+#endif // RBD_MIRROR_IMAGE_SYNC_SYNC_POINT_PRUNE_REQUEST_H
diff --git a/src/tools/rbd_mirror/image_sync/Types.h b/src/tools/rbd_mirror/image_sync/Types.h
new file mode 100644
index 000000000..d748dc93e
--- /dev/null
+++ b/src/tools/rbd_mirror/image_sync/Types.h
@@ -0,0 +1,74 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RBD_MIRROR_IMAGE_SYNC_TYPES_H
+#define RBD_MIRROR_IMAGE_SYNC_TYPES_H
+
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/Types.h"
+#include <list>
+#include <string>
+#include <boost/optional.hpp>
+
+struct Context;
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+
+struct SyncPoint {
+  typedef boost::optional<uint64_t> ObjectNumber;
+
+  SyncPoint() {
+  }
+  SyncPoint(const cls::rbd::SnapshotNamespace& snap_namespace,
+            const std::string& snap_name,
+            const std::string& from_snap_name,
+            const ObjectNumber& object_number)
+    : snap_namespace(snap_namespace), snap_name(snap_name),
+      from_snap_name(from_snap_name), object_number(object_number) {
+  }
+
+  cls::rbd::SnapshotNamespace snap_namespace =
+    {cls::rbd::UserSnapshotNamespace{}};
+  std::string snap_name;
+  std::string from_snap_name;
+  ObjectNumber object_number = boost::none;
+
+  bool operator==(const SyncPoint& rhs) const {
+    return (snap_namespace == rhs.snap_namespace &&
+            snap_name == rhs.snap_name &&
+            from_snap_name == rhs.from_snap_name &&
+            object_number == rhs.object_number);
+  }
+};
+
+typedef std::list<SyncPoint> SyncPoints;
+
+struct SyncPointHandler {
+public:
+  SyncPointHandler(const SyncPointHandler&) = delete;
+  SyncPointHandler& operator=(const SyncPointHandler&) = delete;
+
+  virtual ~SyncPointHandler() {}
+  virtual void destroy() {
+    delete this;
+  }
+
+  virtual SyncPoints get_sync_points() const = 0;
+  virtual librbd::SnapSeqs get_snap_seqs() const = 0;
+
+  virtual void update_sync_points(const librbd::SnapSeqs& snap_seq,
+                                  const SyncPoints& sync_points,
+                                  bool sync_complete,
+                                  Context* on_finish) = 0;
+
+protected:
+  SyncPointHandler() {}
+};
+
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
+
+#endif // RBD_MIRROR_IMAGE_SYNC_TYPES_H
diff --git a/src/tools/rbd_mirror/image_sync/Utils.cc b/src/tools/rbd_mirror/image_sync/Utils.cc
new file mode 100644
index 000000000..6a3eae72d
--- /dev/null
+++ b/src/tools/rbd_mirror/image_sync/Utils.cc
@@ -0,0 +1,24 @@
+// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "Utils.h"
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+namespace util {
+
+namespace {
+
+static const std::string SNAP_NAME_PREFIX(".rbd-mirror");
+
+} // anonymous namespace
+
+std::string get_snapshot_name_prefix(const std::string& local_mirror_uuid) {
+  return SNAP_NAME_PREFIX + "." + local_mirror_uuid + ".";
+}
+
+} // namespace util
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd
diff --git a/src/tools/rbd_mirror/image_sync/Utils.h b/src/tools/rbd_mirror/image_sync/Utils.h
new file mode 100644
index 000000000..139699daa
--- /dev/null
+++ b/src/tools/rbd_mirror/image_sync/Utils.h
@@ -0,0 +1,16 @@
+// -*- mode:c++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <string>
+
+namespace rbd {
+namespace mirror {
+namespace image_sync {
+namespace util {
+
+std::string get_snapshot_name_prefix(const std::string& local_mirror_uuid);
+
+} // namespace util
+} // namespace image_sync
+} // namespace mirror
+} // namespace rbd