Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 1924 insertions, 0 deletions

diff --git a/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc b/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc
new file mode 100644
index 000000000..19a98804c
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc
@@ -0,0 +1,299 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "librbd/ExclusiveLock.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/Operations.h"
+#include "librbd/Utils.h"
+#include "librbd/journal/Policy.h"
+#include "tools/rbd_mirror/image_deleter/Types.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_deleter::SnapshotPurgeRequest: " \
+                           << this << " " << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+using librbd::util::create_context_callback;
+
+template <typename I>
+void SnapshotPurgeRequest<I>::send() {
+  open_image();
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::open_image() {
+  dout(10) << dendl;
+  m_image_ctx = I::create("", m_image_id, nullptr, m_io_ctx, false);
+
+  // ensure non-primary images can be modified
+  m_image_ctx->read_only_mask &= ~librbd::IMAGE_READ_ONLY_FLAG_NON_PRIMARY;
+
+  {
+    std::unique_lock image_locker{m_image_ctx->image_lock};
+    m_image_ctx->set_journal_policy(new JournalPolicy());
+  }
+
+  Context *ctx = create_context_callback<
+    SnapshotPurgeRequest<I>, &SnapshotPurgeRequest<I>::handle_open_image>(
+      this);
+  m_image_ctx->state->open(librbd::OPEN_FLAG_SKIP_OPEN_PARENT, ctx);
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::handle_open_image(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    derr << "failed to open image '" << m_image_id << "': " << cpp_strerror(r)
+         << dendl;
+    m_image_ctx = nullptr;
+
+    finish(r);
+    return;
+  }
+
+  acquire_lock();
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::acquire_lock() {
+  dout(10) << dendl;
+
+  m_image_ctx->owner_lock.lock_shared();
+  if (m_image_ctx->exclusive_lock == nullptr) {
+    m_image_ctx->owner_lock.unlock_shared();
+
+    start_snap_unprotect();
+    return;
+  }
+
+  m_image_ctx->exclusive_lock->acquire_lock(create_context_callback<
+    SnapshotPurgeRequest<I>, &SnapshotPurgeRequest<I>::handle_acquire_lock>(
+      this));
+  m_image_ctx->owner_lock.unlock_shared();
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::handle_acquire_lock(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    derr << "failed to acquire exclusive lock: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  start_snap_unprotect();
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::start_snap_unprotect() {
+  dout(10) << dendl;
+
+  {
+    std::shared_lock image_locker{m_image_ctx->image_lock};
+    m_snaps = m_image_ctx->snaps;
+  }
+  snap_unprotect();
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::snap_unprotect() {
+  if (m_snaps.empty()) {
+    close_image();
+    return;
+  }
+
+  librados::snap_t snap_id = m_snaps.back();
+  m_image_ctx->image_lock.lock_shared();
+  int r = m_image_ctx->get_snap_namespace(snap_id, &m_snap_namespace);
+  if (r < 0) {
+    m_image_ctx->image_lock.unlock_shared();
+
+    derr << "failed to get snap namespace: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  r = m_image_ctx->get_snap_name(snap_id, &m_snap_name);
+  if (r < 0) {
+    m_image_ctx->image_lock.unlock_shared();
+
+    derr << "failed to get snap name: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  bool is_protected;
+  r = m_image_ctx->is_snap_protected(snap_id, &is_protected);
+  if (r < 0) {
+    m_image_ctx->image_lock.unlock_shared();
+
+    derr << "failed to get snap protection status: " << cpp_strerror(r)
+         << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+  m_image_ctx->image_lock.unlock_shared();
+
+  if (!is_protected) {
+    snap_remove();
+    return;
+  }
+
+  dout(10) << "snap_id=" << snap_id << ", "
+           << "snap_namespace=" << m_snap_namespace << ", "
+           << "snap_name=" << m_snap_name << dendl;
+
+  auto finish_op_ctx = start_lock_op(&r);
+  if (finish_op_ctx == nullptr) {
+    derr << "lost exclusive lock" << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  auto ctx = new LambdaContext([this, finish_op_ctx](int r) {
+      handle_snap_unprotect(r);
+      finish_op_ctx->complete(0);
+    });
+  std::shared_lock owner_locker{m_image_ctx->owner_lock};
+  m_image_ctx->operations->execute_snap_unprotect(
+    m_snap_namespace, m_snap_name.c_str(), ctx);
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::handle_snap_unprotect(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -EBUSY) {
+    dout(10) << "snapshot in-use" << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  } else if (r < 0) {
+    derr << "failed to unprotect snapshot: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  {
+    // avoid the need to refresh to delete the newly unprotected snapshot
+    std::shared_lock image_locker{m_image_ctx->image_lock};
+    librados::snap_t snap_id = m_snaps.back();
+    auto snap_info_it = m_image_ctx->snap_info.find(snap_id);
+    if (snap_info_it != m_image_ctx->snap_info.end()) {
+      snap_info_it->second.protection_status =
+        RBD_PROTECTION_STATUS_UNPROTECTED;
+    }
+  }
+
+  snap_remove();
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::snap_remove() {
+  librados::snap_t snap_id = m_snaps.back();
+  dout(10) << "snap_id=" << snap_id << ", "
+           << "snap_namespace=" << m_snap_namespace << ", "
+           << "snap_name=" << m_snap_name << dendl;
+
+  int r;
+  auto finish_op_ctx = start_lock_op(&r);
+  if (finish_op_ctx == nullptr) {
+    derr << "lost exclusive lock" << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  auto ctx = new LambdaContext([this, finish_op_ctx](int r) {
+      handle_snap_remove(r);
+      finish_op_ctx->complete(0);
+    });
+  std::shared_lock owner_locker{m_image_ctx->owner_lock};
+  m_image_ctx->operations->execute_snap_remove(
+    m_snap_namespace, m_snap_name.c_str(), ctx);
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::handle_snap_remove(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -EBUSY) {
+    dout(10) << "snapshot in-use" << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  } else if (r < 0) {
+    derr << "failed to remove snapshot: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  m_snaps.pop_back();
+  snap_unprotect();
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::close_image() {
+  dout(10) << dendl;
+
+  m_image_ctx->state->close(create_context_callback<
+    SnapshotPurgeRequest<I>,
+    &SnapshotPurgeRequest<I>::handle_close_image>(this));
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::handle_close_image(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  m_image_ctx = nullptr;
+
+  if (r < 0) {
+    derr << "failed to close: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+  finish(0);
+}
+
+template <typename I>
+void SnapshotPurgeRequest<I>::finish(int r) {
+  if (m_ret_val < 0) {
+    r = m_ret_val;
+  }
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+template <typename I>
+Context *SnapshotPurgeRequest<I>::start_lock_op(int* r) {
+  std::shared_lock owner_locker{m_image_ctx->owner_lock};
+  if (m_image_ctx->exclusive_lock == nullptr) {
+    return new LambdaContext([](int r) {});
+  }
+  return m_image_ctx->exclusive_lock->start_op(r);
+}
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_deleter::SnapshotPurgeRequest<librbd::ImageCtx>;
diff --git a/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h b/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h
new file mode 100644
index 000000000..70cae8518
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h
@@ -0,0 +1,105 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_RBD_MIRROR_IMAGE_DELETER_SNAPSHOT_PURGE_REQUEST_H
+#define CEPH_RBD_MIRROR_IMAGE_DELETER_SNAPSHOT_PURGE_REQUEST_H
+
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include <string>
+#include <vector>
+
+class Context;
+namespace librbd { struct ImageCtx; }
+
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class SnapshotPurgeRequest {
+public:
+  static SnapshotPurgeRequest* create(librados::IoCtx &io_ctx,
+                                      const std::string &image_id,
+                                      Context *on_finish) {
+    return new SnapshotPurgeRequest(io_ctx, image_id, on_finish);
+  }
+
+  SnapshotPurgeRequest(librados::IoCtx &io_ctx, const std::string &image_id,
+                       Context *on_finish)
+    : m_io_ctx(io_ctx), m_image_id(image_id), m_on_finish(on_finish) {
+  }
+
+  void send();
+
+private:
+  /*
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * OPEN_IMAGE
+   *    |
+   *    v
+   * ACQUIRE_LOCK
+   *    |
+   *    | (repeat for each snapshot)
+   *    |/------------------------\
+   *    |                         |
+   *    v (skip if not needed)    |
+   * SNAP_UNPROTECT               |
+   *    |                         |
+   *    v (skip if not needed)    |
+   * SNAP_REMOVE -----------------/
+   *    |
+   *    v
+   * CLOSE_IMAGE
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  librados::IoCtx &m_io_ctx;
+  std::string m_image_id;
+  Context *m_on_finish;
+
+  ImageCtxT *m_image_ctx = nullptr;
+  int m_ret_val = 0;
+
+  std::vector<librados::snap_t> m_snaps;
+  cls::rbd::SnapshotNamespace m_snap_namespace;
+  std::string m_snap_name;
+
+  void open_image();
+  void handle_open_image(int r);
+
+  void acquire_lock();
+  void handle_acquire_lock(int r);
+
+  void start_snap_unprotect();
+  void snap_unprotect();
+  void handle_snap_unprotect(int r);
+
+  void snap_remove();
+  void handle_snap_remove(int r);
+
+  void close_image();
+  void handle_close_image(int r);
+
+  void finish(int r);
+
+  Context *start_lock_op(int* r);
+
+};
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_deleter::SnapshotPurgeRequest<librbd::ImageCtx>;
+
+#endif // CEPH_RBD_MIRROR_IMAGE_DELETER_SNAPSHOT_PURGE_REQUEST_H
+
diff --git a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc
new file mode 100644
index 000000000..e53923ef3
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc
@@ -0,0 +1,419 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h"
+#include "include/rbd_types.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/WorkQueue.h"
+#include "librbd/ExclusiveLock.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/ImageState.h"
+#include "librbd/Journal.h"
+#include "librbd/TrashWatcher.h"
+#include "librbd/Utils.h"
+#include "librbd/asio/ContextWQ.h"
+#include "librbd/journal/ResetRequest.h"
+#include "librbd/mirror/ImageRemoveRequest.h"
+#include "librbd/mirror/GetInfoRequest.h"
+#include "librbd/trash/MoveRequest.h"
+#include "tools/rbd_mirror/image_deleter/Types.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_deleter::TrashMoveRequest: " \
+                           << this << " " << __func__ << ": "
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+void TrashMoveRequest<I>::send() {
+  get_mirror_image_id();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::get_mirror_image_id() {
+  dout(10) << dendl;
+
+  librados::ObjectReadOperation op;
+  librbd::cls_client::mirror_image_get_image_id_start(&op, m_global_image_id);
+
+  auto aio_comp = create_rados_callback<
+    TrashMoveRequest<I>,
+    &TrashMoveRequest<I>::handle_get_mirror_image_id>(this);
+  m_out_bl.clear();
+  int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_get_mirror_image_id(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == 0) {
+    auto bl_it = m_out_bl.cbegin();
+    r = librbd::cls_client::mirror_image_get_image_id_finish(&bl_it,
+                                                             &m_image_id);
+  }
+  if (r == -ENOENT) {
+    dout(10) << "image " << m_global_image_id << " is not mirrored" << dendl;
+    finish(r);
+    return;
+  } else if (r < 0) {
+    derr << "error retrieving local id for image " << m_global_image_id << ": "
+         << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  get_mirror_info();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::get_mirror_info() {
+  dout(10) << dendl;
+
+  auto ctx = create_context_callback<
+    TrashMoveRequest<I>, &TrashMoveRequest<I>::handle_get_mirror_info>(this);
+  auto req = librbd::mirror::GetInfoRequest<I>::create(
+    m_io_ctx, m_op_work_queue, m_image_id, &m_mirror_image, &m_promotion_state,
+    &m_primary_mirror_uuid, ctx);
+  req->send();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_get_mirror_info(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    dout(5) << "image " << m_global_image_id << " is not mirrored" << dendl;
+    finish(r);
+    return;
+  } else if (r < 0) {
+    derr << "error retrieving image primary info for image "
+         << m_global_image_id << ": " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  if (m_promotion_state == librbd::mirror::PROMOTION_STATE_PRIMARY) {
+    dout(10) << "image " << m_global_image_id << " is local primary" << dendl;
+    finish(-EPERM);
+    return;
+  } else if (m_promotion_state == librbd::mirror::PROMOTION_STATE_ORPHAN &&
+             !m_resync) {
+    dout(10) << "image " << m_global_image_id << " is orphaned" << dendl;
+    finish(-EPERM);
+    return;
+  }
+
+  disable_mirror_image();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::disable_mirror_image() {
+  dout(10) << dendl;
+
+  m_mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING;
+
+  librados::ObjectWriteOperation op;
+  librbd::cls_client::mirror_image_set(&op, m_image_id, m_mirror_image);
+
+  auto aio_comp = create_rados_callback<
+    TrashMoveRequest<I>,
+    &TrashMoveRequest<I>::handle_disable_mirror_image>(this);
+  int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_disable_mirror_image(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    dout(10) << "local image is not mirrored, aborting deletion." << dendl;
+    finish(r);
+    return;
+  } else if (r == -EEXIST || r == -EINVAL) {
+    derr << "cannot disable mirroring for image " << m_global_image_id
+         << ": global_image_id has changed/reused: "
+         << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  } else if (r < 0) {
+    derr << "cannot disable mirroring for image " << m_global_image_id
+         << ": " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  open_image();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::open_image() {
+  dout(10) << dendl;
+
+  m_image_ctx = I::create("", m_image_id, nullptr, m_io_ctx, false);
+
+  // ensure non-primary images can be modified
+  m_image_ctx->read_only_mask &= ~librbd::IMAGE_READ_ONLY_FLAG_NON_PRIMARY;
+
+  {
+    // don't attempt to open the journal
+    std::unique_lock image_locker{m_image_ctx->image_lock};
+    m_image_ctx->set_journal_policy(new JournalPolicy());
+  }
+
+  Context *ctx = create_context_callback<
+    TrashMoveRequest<I>, &TrashMoveRequest<I>::handle_open_image>(this);
+  m_image_ctx->state->open(librbd::OPEN_FLAG_SKIP_OPEN_PARENT, ctx);
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_open_image(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    dout(5) << "mirror image does not exist, removing orphaned metadata" << dendl;
+    m_image_ctx = nullptr;
+    remove_mirror_image();
+    return;
+  }
+
+  if (r < 0) {
+    derr << "failed to open image: " << cpp_strerror(r) << dendl;
+    m_image_ctx = nullptr;
+    finish(r);
+    return;
+  }
+
+  if (m_image_ctx->old_format) {
+    derr << "cannot move v1 image to trash" << dendl;
+    m_ret_val = -EINVAL;
+    close_image();
+    return;
+  }
+
+  reset_journal();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::reset_journal() {
+  if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+    // snapshot-based mirroring doesn't require journal feature
+    acquire_lock();
+    return;
+  }
+
+  dout(10) << dendl;
+
+  // TODO use Journal thread pool for journal ops until converted to ASIO
+  ContextWQ* context_wq;
+  librbd::Journal<>::get_work_queue(
+    reinterpret_cast<CephContext*>(m_io_ctx.cct()), &context_wq);
+
+  // ensure that if the image is recovered any peers will split-brain
+  auto ctx = create_context_callback<
+    TrashMoveRequest<I>, &TrashMoveRequest<I>::handle_reset_journal>(this);
+  auto req = librbd::journal::ResetRequest<I>::create(
+    m_io_ctx, m_image_id, librbd::Journal<>::IMAGE_CLIENT_ID,
+    librbd::Journal<>::LOCAL_MIRROR_UUID, context_wq, ctx);
+  req->send();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_reset_journal(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r < 0 && r != -ENOENT) {
+    derr << "failed to reset journal: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  acquire_lock();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::acquire_lock() {
+  m_image_ctx->owner_lock.lock_shared();
+  if (m_image_ctx->exclusive_lock == nullptr) {
+    m_image_ctx->owner_lock.unlock_shared();
+
+    if (m_mirror_image.mode == cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT) {
+      // snapshot-based mirroring doesn't require exclusive-lock
+      trash_move();
+    } else {
+      derr << "exclusive lock feature not enabled" << dendl;
+      m_ret_val = -EINVAL;
+      close_image();
+    }
+    return;
+  }
+
+  dout(10) << dendl;
+
+  Context *ctx = create_context_callback<
+    TrashMoveRequest<I>, &TrashMoveRequest<I>::handle_acquire_lock>(this);
+  m_image_ctx->exclusive_lock->block_requests(0);
+  m_image_ctx->exclusive_lock->acquire_lock(ctx);
+  m_image_ctx->owner_lock.unlock_shared();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_acquire_lock(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    derr << "failed to acquire exclusive lock: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  trash_move();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::trash_move() {
+  dout(10) << dendl;
+
+  utime_t delete_time{ceph_clock_now()};
+  utime_t deferment_end_time{delete_time};
+  deferment_end_time +=
+    m_image_ctx->config.template get_val<uint64_t>("rbd_mirroring_delete_delay");
+
+  m_trash_image_spec = {
+    cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, m_image_ctx->name, delete_time,
+    deferment_end_time};
+
+  Context *ctx = create_context_callback<
+    TrashMoveRequest<I>, &TrashMoveRequest<I>::handle_trash_move>(this);
+  auto req = librbd::trash::MoveRequest<I>::create(
+    m_io_ctx, m_image_id, m_trash_image_spec, ctx);
+  req->send();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_trash_move(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    derr << "failed to move image to trash: " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+    close_image();
+    return;
+  }
+
+  m_moved_to_trash = true;
+  remove_mirror_image();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::remove_mirror_image() {
+  dout(10) << dendl;
+
+  auto ctx = create_context_callback<
+    TrashMoveRequest<I>,
+    &TrashMoveRequest<I>::handle_remove_mirror_image>(this);
+  auto req = librbd::mirror::ImageRemoveRequest<I>::create(
+    m_io_ctx, m_global_image_id, m_image_id, ctx);
+  req->send();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_remove_mirror_image(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    dout(10) << "local image is not mirrored" << dendl;
+  } else if (r < 0) {
+    derr << "failed to remove mirror image state for " << m_global_image_id
+         << ": " << cpp_strerror(r) << dendl;
+    m_ret_val = r;
+  }
+
+  close_image();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::close_image() {
+  dout(10) << dendl;
+
+  if (m_image_ctx == nullptr) {
+    handle_close_image(0);
+    return;
+  }
+  Context *ctx = create_context_callback<
+    TrashMoveRequest<I>, &TrashMoveRequest<I>::handle_close_image>(this);
+  m_image_ctx->state->close(ctx);
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_close_image(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  m_image_ctx = nullptr;
+
+  if (r < 0) {
+    derr << "failed to close image: " << cpp_strerror(r) << dendl;
+  }
+
+  // don't send notification if we failed
+  if (!m_moved_to_trash) {
+    finish(0);
+    return;
+  }
+
+  notify_trash_add();
+}
+
+template <typename I>
+void TrashMoveRequest<I>::notify_trash_add() {
+  dout(10) << dendl;
+
+  Context *ctx = create_context_callback<
+    TrashMoveRequest<I>, &TrashMoveRequest<I>::handle_notify_trash_add>(this);
+  librbd::TrashWatcher<I>::notify_image_added(m_io_ctx, m_image_id,
+                                              m_trash_image_spec, ctx);
+}
+
+template <typename I>
+void TrashMoveRequest<I>::handle_notify_trash_add(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    derr << "failed to notify trash watchers: " << cpp_strerror(r) << dendl;
+  }
+
+  finish(0);
+}
+
+template <typename I>
+void TrashMoveRequest<I>::finish(int r) {
+  if (m_ret_val < 0) {
+    r = m_ret_val;
+  }
+
+  dout(10) << "r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_deleter::TrashMoveRequest<librbd::ImageCtx>;
+
diff --git a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.h b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.h
new file mode 100644
index 000000000..5b3f02519
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.h
@@ -0,0 +1,142 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_MOVE_REQUEST_H
+#define CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_MOVE_REQUEST_H
+
+#include "include/buffer.h"
+#include "include/rados/librados.hpp"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/mirror/Types.h"
+#include <string>
+
+struct Context;
+namespace librbd {
+struct ImageCtx;
+namespace asio { struct ContextWQ; }
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class TrashMoveRequest {
+public:
+  static TrashMoveRequest* create(librados::IoCtx& io_ctx,
+                                  const std::string& global_image_id,
+                                  bool resync,
+                                  librbd::asio::ContextWQ* op_work_queue,
+                                  Context* on_finish) {
+    return new TrashMoveRequest(io_ctx, global_image_id, resync, op_work_queue,
+                                on_finish);
+  }
+
+  TrashMoveRequest(librados::IoCtx& io_ctx, const std::string& global_image_id,
+                   bool resync, librbd::asio::ContextWQ* op_work_queue,
+                   Context* on_finish)
+    : m_io_ctx(io_ctx), m_global_image_id(global_image_id), m_resync(resync),
+      m_op_work_queue(op_work_queue), m_on_finish(on_finish) {
+  }
+
+  void send();
+
+private:
+  /*
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * GET_MIRROR_IMAGE_ID
+   *    |
+   *    v
+   * GET_MIRROR_INFO
+   *    |
+   *    v
+   * DISABLE_MIRROR_IMAGE
+   *    |
+   *    v
+   * OPEN_IMAGE
+   *    |
+   *    v (skip if not needed)
+   * RESET_JOURNAL
+   *    |
+   *    v (skip if not needed)
+   * ACQUIRE_LOCK
+   *    |
+   *    v
+   * TRASH_MOVE
+   *    |
+   *    v
+   * REMOVE_MIRROR_IMAGE
+   *    |
+   *    v
+   * CLOSE_IMAGE
+   *    |
+   *    v
+   * NOTIFY_TRASH_ADD
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  librados::IoCtx &m_io_ctx;
+  std::string m_global_image_id;
+  bool m_resync;
+  librbd::asio::ContextWQ *m_op_work_queue;
+  Context *m_on_finish;
+
+  ceph::bufferlist m_out_bl;
+  std::string m_image_id;
+  cls::rbd::MirrorImage m_mirror_image;
+  librbd::mirror::PromotionState m_promotion_state;
+  std::string m_primary_mirror_uuid;
+  cls::rbd::TrashImageSpec m_trash_image_spec;
+  ImageCtxT *m_image_ctx = nullptr;;
+  int m_ret_val = 0;
+  bool m_moved_to_trash = false;
+
+  void get_mirror_image_id();
+  void handle_get_mirror_image_id(int r);
+
+  void get_mirror_info();
+  void handle_get_mirror_info(int r);
+
+  void disable_mirror_image();
+  void handle_disable_mirror_image(int r);
+
+  void open_image();
+  void handle_open_image(int r);
+
+  void reset_journal();
+  void handle_reset_journal(int r);
+
+  void acquire_lock();
+  void handle_acquire_lock(int r);
+
+  void trash_move();
+  void handle_trash_move(int r);
+
+  void remove_mirror_image();
+  void handle_remove_mirror_image(int r);
+
+  void close_image();
+  void handle_close_image(int r);
+
+  void notify_trash_add();
+  void handle_notify_trash_add(int r);
+
+  void finish(int r);
+
+};
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_deleter::TrashMoveRequest<librbd::ImageCtx>;
+
+#endif // CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H
diff --git a/src/tools/rbd_mirror/image_deleter/TrashRemoveRequest.cc b/src/tools/rbd_mirror/image_deleter/TrashRemoveRequest.cc
new file mode 100644
index 000000000..4d7c1c9df
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/TrashRemoveRequest.cc
@@ -0,0 +1,265 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd_mirror/image_deleter/TrashRemoveRequest.h"
+#include "include/ceph_assert.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/WorkQueue.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/TrashWatcher.h"
+#include "librbd/Utils.h"
+#include "librbd/asio/ContextWQ.h"
+#include "librbd/trash/RemoveRequest.h"
+#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_deleter::TrashRemoveRequest: " \
+                           << this << " " << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+template <typename I>
+void TrashRemoveRequest<I>::send() {
+  *m_error_result = ERROR_RESULT_RETRY;
+
+  get_trash_image_spec();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::get_trash_image_spec() {
+  dout(10) << dendl;
+
+  librados::ObjectReadOperation op;
+  librbd::cls_client::trash_get_start(&op, m_image_id);
+
+  auto aio_comp = create_rados_callback<
+    TrashRemoveRequest<I>,
+    &TrashRemoveRequest<I>::handle_get_trash_image_spec>(this);
+  m_out_bl.clear();
+  int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::handle_get_trash_image_spec(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == 0) {
+    auto bl_it = m_out_bl.cbegin();
+    r = librbd::cls_client::trash_get_finish(&bl_it, &m_trash_image_spec);
+  }
+
+  if (r == -ENOENT || (r >= 0 && m_trash_image_spec.source !=
+                                   cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING)) {
+    dout(10) << "image id " << m_image_id << " not in mirroring trash" << dendl;
+    finish(0);
+    return;
+  } else if (r < 0) {
+    derr << "error getting image id " << m_image_id << " info from trash: "
+         << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  if (m_trash_image_spec.state != cls::rbd::TRASH_IMAGE_STATE_NORMAL &&
+      m_trash_image_spec.state != cls::rbd::TRASH_IMAGE_STATE_REMOVING) {
+    dout(10) << "image " << m_image_id << " is not in an expected trash state: "
+             << m_trash_image_spec.state << dendl;
+    *m_error_result = ERROR_RESULT_RETRY_IMMEDIATELY;
+    finish(-EBUSY);
+    return;
+  }
+
+  set_trash_state();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::set_trash_state() {
+  if (m_trash_image_spec.state == cls::rbd::TRASH_IMAGE_STATE_REMOVING) {
+    get_snap_context();
+    return;
+  }
+
+  dout(10) << dendl;
+
+  librados::ObjectWriteOperation op;
+  librbd::cls_client::trash_state_set(&op, m_image_id,
+                                      cls::rbd::TRASH_IMAGE_STATE_REMOVING,
+                                      cls::rbd::TRASH_IMAGE_STATE_NORMAL);
+
+  auto aio_comp = create_rados_callback<
+    TrashRemoveRequest<I>,
+    &TrashRemoveRequest<I>::handle_set_trash_state>(this);
+  int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::handle_set_trash_state(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -ENOENT) {
+    dout(10) << "image id " << m_image_id << " not in mirroring trash" << dendl;
+    finish(0);
+    return;
+  } else if (r < 0 && r != -EOPNOTSUPP) {
+    derr << "error setting trash image state for image id " << m_image_id
+         << ": " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  get_snap_context();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::get_snap_context() {
+  dout(10) << dendl;
+
+  librados::ObjectReadOperation op;
+  librbd::cls_client::get_snapcontext_start(&op);
+
+  std::string header_oid = librbd::util::header_name(m_image_id);
+
+  auto aio_comp = create_rados_callback<
+    TrashRemoveRequest<I>,
+    &TrashRemoveRequest<I>::handle_get_snap_context>(this);
+  m_out_bl.clear();
+  int r = m_io_ctx.aio_operate(header_oid, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::handle_get_snap_context(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  ::SnapContext snapc;
+  if (r == 0) {
+    auto bl_it = m_out_bl.cbegin();
+    r = librbd::cls_client::get_snapcontext_finish(&bl_it, &snapc);
+  }
+  if (r < 0 && r != -ENOENT) {
+    derr << "error retrieving snapshot context for image "
+         << m_image_id << ": " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  m_has_snapshots = (!snapc.empty());
+  purge_snapshots();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::purge_snapshots() {
+  if (!m_has_snapshots) {
+    remove_image();
+    return;
+  }
+
+  dout(10) << dendl;
+  auto ctx = create_context_callback<
+    TrashRemoveRequest<I>,
+    &TrashRemoveRequest<I>::handle_purge_snapshots>(this);
+  auto req = SnapshotPurgeRequest<I>::create(m_io_ctx, m_image_id, ctx);
+  req->send();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::handle_purge_snapshots(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r == -EBUSY) {
+    dout(10) << "snapshots still in-use" << dendl;
+    *m_error_result = ERROR_RESULT_RETRY_IMMEDIATELY;
+    finish(r);
+    return;
+  } else if (r < 0) {
+    derr << "failed to purge image snapshots: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  remove_image();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::remove_image() {
+  dout(10) << dendl;
+
+  auto ctx = create_context_callback<
+    TrashRemoveRequest<I>,
+    &TrashRemoveRequest<I>::handle_remove_image>(this);
+  auto req = librbd::trash::RemoveRequest<I>::create(
+    m_io_ctx, m_image_id, m_op_work_queue, true, m_progress_ctx,
+    ctx);
+  req->send();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::handle_remove_image(int r) {
+  dout(10) << "r=" << r << dendl;
+  if (r == -ENOTEMPTY) {
+    // image must have clone v2 snapshot still associated to child
+    dout(10) << "snapshots still in-use" << dendl;
+    *m_error_result = ERROR_RESULT_RETRY_IMMEDIATELY;
+    finish(-EBUSY);
+    return;
+  }
+
+  if (r < 0 && r != -ENOENT) {
+    derr << "error removing image " << m_image_id << " "
+         << "(" << m_image_id << ") from local pool: "
+         << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  notify_trash_removed();
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::notify_trash_removed() {
+  dout(10) << dendl;
+
+  Context *ctx = create_context_callback<
+    TrashRemoveRequest<I>,
+    &TrashRemoveRequest<I>::handle_notify_trash_removed>(this);
+  librbd::TrashWatcher<I>::notify_image_removed(m_io_ctx, m_image_id, ctx);
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::handle_notify_trash_removed(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  if (r < 0) {
+    derr << "failed to notify trash watchers: " << cpp_strerror(r) << dendl;
+  }
+
+  finish(0);
+}
+
+template <typename I>
+void TrashRemoveRequest<I>::finish(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_deleter::TrashRemoveRequest<librbd::ImageCtx>;
diff --git a/src/tools/rbd_mirror/image_deleter/TrashRemoveRequest.h b/src/tools/rbd_mirror/image_deleter/TrashRemoveRequest.h
new file mode 100644
index 000000000..b99736b33
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/TrashRemoveRequest.h
@@ -0,0 +1,117 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_RBD_MIRROR_IMAGE_DELETER_TRASH_REMOVE_REQUEST_H
+#define CEPH_RBD_MIRROR_IMAGE_DELETER_TRASH_REMOVE_REQUEST_H
+
+#include "include/rados/librados.hpp"
+#include "include/buffer.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/internal.h"
+#include "tools/rbd_mirror/image_deleter/Types.h"
+#include <string>
+#include <vector>
+
+class Context;
+class ContextWQ;
+namespace librbd {
+struct ImageCtx;
+namespace asio { struct ContextWQ; }
+} // namespace librbd
+
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class TrashRemoveRequest {
+public:
+  static TrashRemoveRequest* create(librados::IoCtx &io_ctx,
+                                    const std::string &image_id,
+                                    ErrorResult *error_result,
+                                    librbd::asio::ContextWQ *op_work_queue,
+                                    Context *on_finish) {
+    return new TrashRemoveRequest(io_ctx, image_id, error_result, op_work_queue,
+                                  on_finish);
+  }
+
+  TrashRemoveRequest(librados::IoCtx &io_ctx, const std::string &image_id,
+                     ErrorResult *error_result,
+                     librbd::asio::ContextWQ *op_work_queue,
+                     Context *on_finish)
+    : m_io_ctx(io_ctx), m_image_id(image_id), m_error_result(error_result),
+      m_op_work_queue(op_work_queue), m_on_finish(on_finish) {
+  }
+
+  void send();
+
+private:
+  /*
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * GET_TRASH_IMAGE_SPEC
+   *    |
+   *    v
+   * SET_TRASH_STATE
+   *    |
+   *    v
+   * GET_SNAP_CONTEXT
+   *    |
+   *    v
+   * PURGE_SNAPSHOTS
+   *    |
+   *    v
+   * TRASH_REMOVE
+   *    |
+   *    v
+   * NOTIFY_TRASH_REMOVE
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  librados::IoCtx &m_io_ctx;
+  std::string m_image_id;
+  ErrorResult *m_error_result;
+  librbd::asio::ContextWQ *m_op_work_queue;
+  Context *m_on_finish;
+
+  ceph::bufferlist m_out_bl;
+  cls::rbd::TrashImageSpec m_trash_image_spec;
+  bool m_has_snapshots = false;
+  librbd::NoOpProgressContext m_progress_ctx;
+
+  void get_trash_image_spec();
+  void handle_get_trash_image_spec(int r);
+
+  void set_trash_state();
+  void handle_set_trash_state(int r);
+
+  void get_snap_context();
+  void handle_get_snap_context(int r);
+
+  void purge_snapshots();
+  void handle_purge_snapshots(int r);
+
+  void remove_image();
+  void handle_remove_image(int r);
+
+  void notify_trash_removed();
+  void handle_notify_trash_removed(int r);
+
+  void finish(int r);
+
+};
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_deleter::TrashRemoveRequest<librbd::ImageCtx>;
+
+#endif // CEPH_RBD_MIRROR_IMAGE_DELETER_TRASH_REMOVE_REQUEST_H
diff --git a/src/tools/rbd_mirror/image_deleter/TrashWatcher.cc b/src/tools/rbd_mirror/image_deleter/TrashWatcher.cc
new file mode 100644
index 000000000..552d77e0e
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/TrashWatcher.cc
@@ -0,0 +1,384 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "tools/rbd_mirror/image_deleter/TrashWatcher.h"
+#include "include/rbd_types.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "common/debug.h"
+#include "common/errno.h"
+#include "common/Timer.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "librbd/asio/ContextWQ.h"
+#include "tools/rbd_mirror/Threads.h"
+#include "tools/rbd_mirror/image_deleter/Types.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::image_deleter::TrashWatcher: " \
+                           << this << " " << __func__ << ": "
+
+using librbd::util::create_context_callback;
+using librbd::util::create_rados_callback;
+
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+namespace {
+
+const size_t MAX_RETURN = 1024;
+
+} // anonymous namespace
+
+template <typename I>
+TrashWatcher<I>::TrashWatcher(librados::IoCtx &io_ctx, Threads<I> *threads,
+                              TrashListener& trash_listener)
+  : librbd::TrashWatcher<I>(io_ctx, threads->work_queue),
+    m_io_ctx(io_ctx), m_threads(threads), m_trash_listener(trash_listener),
+    m_lock(ceph::make_mutex(librbd::util::unique_lock_name(
+      "rbd::mirror::image_deleter::TrashWatcher", this))) {
+}
+
+template <typename I>
+void TrashWatcher<I>::init(Context *on_finish) {
+  dout(5) << dendl;
+
+  {
+    std::lock_guard locker{m_lock};
+    m_on_init_finish = on_finish;
+
+    ceph_assert(!m_trash_list_in_progress);
+    m_trash_list_in_progress = true;
+  }
+
+  create_trash();
+}
+
+template <typename I>
+void TrashWatcher<I>::shut_down(Context *on_finish) {
+  dout(5) << dendl;
+
+  {
+    std::scoped_lock locker{m_threads->timer_lock, m_lock};
+
+    ceph_assert(!m_shutting_down);
+    m_shutting_down = true;
+    if (m_timer_ctx != nullptr) {
+      m_threads->timer->cancel_event(m_timer_ctx);
+      m_timer_ctx = nullptr;
+    }
+  }
+
+  auto ctx = new LambdaContext([this, on_finish](int r) {
+      unregister_watcher(on_finish);
+    });
+  m_async_op_tracker.wait_for_ops(ctx);
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_image_added(const std::string &image_id,
+                                         const cls::rbd::TrashImageSpec& spec) {
+  dout(10) << "image_id=" << image_id << dendl;
+
+  std::lock_guard locker{m_lock};
+  add_image(image_id, spec);
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_image_removed(const std::string &image_id) {
+  // ignore removals -- the image deleter will ignore -ENOENTs
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_rewatch_complete(int r) {
+  dout(5) << "r=" << r << dendl;
+
+  if (r == -EBLOCKLISTED) {
+    dout(0) << "detected client is blocklisted" << dendl;
+    return;
+  } else if (r == -ENOENT) {
+    dout(5) << "trash directory deleted" << dendl;
+  } else if (r < 0) {
+    derr << "unexpected error re-registering trash directory watch: "
+         << cpp_strerror(r) << dendl;
+  }
+  schedule_trash_list(30);
+}
+
+template <typename I>
+void TrashWatcher<I>::create_trash() {
+  dout(20) << dendl;
+  {
+    std::lock_guard locker{m_lock};
+    ceph_assert(m_trash_list_in_progress);
+  }
+
+  librados::ObjectWriteOperation op;
+  op.create(false);
+
+  m_async_op_tracker.start_op();
+  auto aio_comp = create_rados_callback<
+    TrashWatcher<I>, &TrashWatcher<I>::handle_create_trash>(this);
+  int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_create_trash(int r) {
+  dout(20) << "r=" << r << dendl;
+  {
+    std::lock_guard locker{m_lock};
+    ceph_assert(m_trash_list_in_progress);
+  }
+
+  Context* on_init_finish = nullptr;
+  if (r == -EBLOCKLISTED || r == -ENOENT) {
+    if (r == -EBLOCKLISTED) {
+      dout(0) << "detected client is blocklisted" << dendl;
+    } else {
+      dout(0) << "detected pool no longer exists" << dendl;
+    }
+
+    std::lock_guard locker{m_lock};
+    std::swap(on_init_finish, m_on_init_finish);
+    m_trash_list_in_progress = false;
+  } else if (r < 0 && r != -EEXIST) {
+    derr << "failed to create trash object: " << cpp_strerror(r) << dendl;
+    {
+      std::lock_guard locker{m_lock};
+      m_trash_list_in_progress = false;
+    }
+
+    schedule_trash_list(30);
+  } else {
+    register_watcher();
+  }
+
+  m_async_op_tracker.finish_op();
+  if (on_init_finish != nullptr) {
+    on_init_finish->complete(r);
+  }
+}
+
+template <typename I>
+void TrashWatcher<I>::register_watcher() {
+  {
+    std::lock_guard locker{m_lock};
+    ceph_assert(m_trash_list_in_progress);
+  }
+
+  // if the watch registration is in-flight, let the watcher
+  // handle the transition -- only (re-)register if it's not registered
+  if (!this->is_unregistered()) {
+    trash_list(true);
+    return;
+  }
+
+  // first time registering or the watch failed
+  dout(5) << dendl;
+  m_async_op_tracker.start_op();
+
+  Context *ctx = create_context_callback<
+    TrashWatcher, &TrashWatcher<I>::handle_register_watcher>(this);
+  this->register_watch(ctx);
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_register_watcher(int r) {
+  dout(5) << "r=" << r << dendl;
+
+  {
+    std::lock_guard locker{m_lock};
+    ceph_assert(m_trash_list_in_progress);
+    if (r < 0) {
+      m_trash_list_in_progress = false;
+    }
+  }
+
+  Context *on_init_finish = nullptr;
+  if (r >= 0) {
+    trash_list(true);
+  } else if (r == -EBLOCKLISTED) {
+    dout(0) << "detected client is blocklisted" << dendl;
+
+    std::lock_guard locker{m_lock};
+    std::swap(on_init_finish, m_on_init_finish);
+  } else {
+    derr << "unexpected error registering trash directory watch: "
+         << cpp_strerror(r) << dendl;
+    schedule_trash_list(10);
+  }
+
+  m_async_op_tracker.finish_op();
+  if (on_init_finish != nullptr) {
+    on_init_finish->complete(r);
+  }
+}
+
+template <typename I>
+void TrashWatcher<I>::unregister_watcher(Context* on_finish) {
+  dout(5) << dendl;
+
+  m_async_op_tracker.start_op();
+  Context *ctx = new LambdaContext([this, on_finish](int r) {
+      handle_unregister_watcher(r, on_finish);
+    });
+  this->unregister_watch(ctx);
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_unregister_watcher(int r, Context* on_finish) {
+  dout(5) << "unregister_watcher: r=" << r << dendl;
+  if (r < 0) {
+    derr << "error unregistering watcher for trash directory: "
+         << cpp_strerror(r) << dendl;
+  }
+  m_async_op_tracker.finish_op();
+  on_finish->complete(0);
+}
+
+template <typename I>
+void TrashWatcher<I>::trash_list(bool initial_request) {
+  if (initial_request) {
+    m_async_op_tracker.start_op();
+    m_last_image_id = "";
+  }
+
+  dout(5) << "last_image_id=" << m_last_image_id << dendl;
+
+  {
+    std::lock_guard locker{m_lock};
+    ceph_assert(m_trash_list_in_progress);
+  }
+
+  librados::ObjectReadOperation op;
+  librbd::cls_client::trash_list_start(&op, m_last_image_id, MAX_RETURN);
+
+  librados::AioCompletion *aio_comp = create_rados_callback<
+    TrashWatcher<I>, &TrashWatcher<I>::handle_trash_list>(this);
+  m_out_bl.clear();
+  int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op, &m_out_bl);
+  ceph_assert(r == 0);
+  aio_comp->release();
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_trash_list(int r) {
+  dout(5) << "r=" << r << dendl;
+
+  std::map<std::string, cls::rbd::TrashImageSpec> images;
+  if (r >= 0) {
+    auto bl_it = m_out_bl.cbegin();
+    r = librbd::cls_client::trash_list_finish(&bl_it, &images);
+  }
+
+  Context *on_init_finish = nullptr;
+  {
+    std::lock_guard locker{m_lock};
+    ceph_assert(m_trash_list_in_progress);
+    if (r >= 0) {
+      for (auto& image : images) {
+        add_image(image.first, image.second);
+      }
+    } else if (r == -ENOENT) {
+      r = 0;
+    }
+
+    if (r == -EBLOCKLISTED) {
+      dout(0) << "detected client is blocklisted during trash refresh" << dendl;
+      m_trash_list_in_progress = false;
+      std::swap(on_init_finish, m_on_init_finish);
+    } else if (r >= 0 && images.size() < MAX_RETURN) {
+      m_trash_list_in_progress = false;
+      std::swap(on_init_finish, m_on_init_finish);
+    } else if (r < 0) {
+      m_trash_list_in_progress = false;
+    }
+  }
+
+  if (r >= 0 && images.size() == MAX_RETURN) {
+    m_last_image_id = images.rbegin()->first;
+    trash_list(false);
+    return;
+  } else if (r < 0 && r != -EBLOCKLISTED) {
+    derr << "failed to retrieve trash directory: " << cpp_strerror(r) << dendl;
+    schedule_trash_list(10);
+  }
+
+  m_async_op_tracker.finish_op();
+  if (on_init_finish != nullptr) {
+    on_init_finish->complete(r);
+  }
+}
+
+template <typename I>
+void TrashWatcher<I>::schedule_trash_list(double interval) {
+  std::scoped_lock locker{m_threads->timer_lock, m_lock};
+  if (m_shutting_down || m_trash_list_in_progress || m_timer_ctx != nullptr) {
+    if (m_trash_list_in_progress && !m_deferred_trash_list) {
+      dout(5) << "deferring refresh until in-flight refresh completes" << dendl;
+      m_deferred_trash_list = true;
+    }
+    return;
+  }
+
+  dout(5) << dendl;
+  m_timer_ctx = m_threads->timer->add_event_after(
+    interval,
+    new LambdaContext([this](int r) {
+	process_trash_list();
+      }));
+}
+
+template <typename I>
+void TrashWatcher<I>::process_trash_list() {
+  dout(5) << dendl;
+
+  ceph_assert(ceph_mutex_is_locked(m_threads->timer_lock));
+  ceph_assert(m_timer_ctx != nullptr);
+  m_timer_ctx = nullptr;
+
+  {
+    std::lock_guard locker{m_lock};
+    ceph_assert(!m_trash_list_in_progress);
+    m_trash_list_in_progress = true;
+  }
+
+  // execute outside of the timer's lock
+  m_async_op_tracker.start_op();
+  Context *ctx = new LambdaContext([this](int r) {
+      create_trash();
+      m_async_op_tracker.finish_op();
+    });
+  m_threads->work_queue->queue(ctx, 0);
+}
+
+template <typename I>
+void TrashWatcher<I>::add_image(const std::string& image_id,
+                                const cls::rbd::TrashImageSpec& spec) {
+  if (spec.source != cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING) {
+    return;
+  }
+
+  ceph_assert(ceph_mutex_is_locked(m_lock));
+  auto& deferment_end_time = spec.deferment_end_time;
+  dout(10) << "image_id=" << image_id << ", "
+           << "deferment_end_time=" << deferment_end_time << dendl;
+
+  m_async_op_tracker.start_op();
+  auto ctx = new LambdaContext([this, image_id, deferment_end_time](int r) {
+      m_trash_listener.handle_trash_image(image_id,
+					  deferment_end_time.to_real_time());
+      m_async_op_tracker.finish_op();
+    });
+  m_threads->work_queue->queue(ctx, 0);
+}
+
+} // namespace image_deleter;
+} // namespace mirror
+} // namespace rbd
+
+template class rbd::mirror::image_deleter::TrashWatcher<librbd::ImageCtx>;
diff --git a/src/tools/rbd_mirror/image_deleter/TrashWatcher.h b/src/tools/rbd_mirror/image_deleter/TrashWatcher.h
new file mode 100644
index 000000000..e818a102c
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/TrashWatcher.h
@@ -0,0 +1,139 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H
+#define CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H
+
+#include "include/rados/librados.hpp"
+#include "common/AsyncOpTracker.h"
+#include "common/ceph_mutex.h"
+#include "librbd/TrashWatcher.h"
+#include <set>
+#include <string>
+
+struct Context;
+namespace librbd { struct ImageCtx; }
+
+namespace rbd {
+namespace mirror {
+
+template <typename> struct Threads;
+
+namespace image_deleter {
+
+struct TrashListener;
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class TrashWatcher : public librbd::TrashWatcher<ImageCtxT> {
+public:
+  static TrashWatcher* create(librados::IoCtx &io_ctx,
+                              Threads<ImageCtxT> *threads,
+                              TrashListener& trash_listener) {
+    return new TrashWatcher(io_ctx, threads, trash_listener);
+  }
+
+  TrashWatcher(librados::IoCtx &io_ctx, Threads<ImageCtxT> *threads,
+               TrashListener& trash_listener);
+  TrashWatcher(const TrashWatcher&) = delete;
+  TrashWatcher& operator=(const TrashWatcher&) = delete;
+
+  void init(Context *on_finish);
+  void shut_down(Context *on_finish);
+
+protected:
+  void handle_image_added(const std::string &image_id,
+                          const cls::rbd::TrashImageSpec& spec) override;
+
+  void handle_image_removed(const std::string &image_id) override;
+
+  void handle_rewatch_complete(int r) override;
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   *  INIT
+   *    |
+   *    v
+   * CREATE_TRASH
+   *    |
+   *    v
+   * REGISTER_WATCHER
+   *    |
+   *    |/--------------------------------\
+   *    |                                 |
+   *    |/---------\                      |
+   *    |          |                      |
+   *    v          | (more images)        |
+   * TRASH_LIST ---/                      |
+   *    |                                 |
+   *    |/----------------------------\   |
+   *    |                             |   |
+   *    v                             |   |
+   * <idle> --\                       |   |
+   *    |     |                       |   |
+   *    |     |\---> IMAGE_ADDED -----/   |
+   *    |     |                           |
+   *    |     \----> WATCH_ERROR ---------/
+   *    v
+   * SHUT_DOWN
+   *    |
+   *    v
+   * UNREGISTER_WATCHER
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  librados::IoCtx m_io_ctx;
+  Threads<ImageCtxT> *m_threads;
+  TrashListener& m_trash_listener;
+
+  std::string m_last_image_id;
+  bufferlist m_out_bl;
+
+  mutable ceph::mutex m_lock;
+
+  Context *m_on_init_finish = nullptr;
+  Context *m_timer_ctx = nullptr;
+
+  AsyncOpTracker m_async_op_tracker;
+  bool m_trash_list_in_progress = false;
+  bool m_deferred_trash_list = false;
+  bool m_shutting_down = false;
+
+  void register_watcher();
+  void handle_register_watcher(int r);
+
+  void create_trash();
+  void handle_create_trash(int r);
+
+  void unregister_watcher(Context* on_finish);
+  void handle_unregister_watcher(int r, Context* on_finish);
+
+  void trash_list(bool initial_request);
+  void handle_trash_list(int r);
+
+  void schedule_trash_list(double interval);
+  void process_trash_list();
+
+  void get_mirror_uuid();
+  void handle_get_mirror_uuid(int r);
+
+  void add_image(const std::string& image_id,
+                 const cls::rbd::TrashImageSpec& spec);
+
+};
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+extern template class rbd::mirror::image_deleter::TrashWatcher<librbd::ImageCtx>;
+
+#endif // CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H
diff --git a/src/tools/rbd_mirror/image_deleter/Types.h b/src/tools/rbd_mirror/image_deleter/Types.h
new file mode 100644
index 000000000..1c70b7e14
--- /dev/null
+++ b/src/tools/rbd_mirror/image_deleter/Types.h
@@ -0,0 +1,54 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H
+#define CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H
+
+#include "include/Context.h"
+#include "librbd/journal/Policy.h"
+#include <string>
+
+struct utime_t;
+
+namespace rbd {
+namespace mirror {
+namespace image_deleter {
+
+enum ErrorResult {
+  ERROR_RESULT_COMPLETE,
+  ERROR_RESULT_RETRY,
+  ERROR_RESULT_RETRY_IMMEDIATELY
+};
+
+struct TrashListener {
+  TrashListener() {
+  }
+  TrashListener(const TrashListener&) = delete;
+  TrashListener& operator=(const TrashListener&) = delete;
+
+  virtual ~TrashListener() {
+  }
+
+  virtual void handle_trash_image(const std::string& image_id,
+    const ceph::real_clock::time_point& deferment_end_time) = 0;
+
+};
+
+struct JournalPolicy : public librbd::journal::Policy {
+  bool append_disabled() const override {
+    return true;
+  }
+  bool journal_disabled() const override {
+    return true;
+  }
+
+  void allocate_tag_on_lock(Context *on_finish) override {
+    on_finish->complete(0);
+  }
+};
+
+} // namespace image_deleter
+} // namespace mirror
+} // namespace rbd
+
+#endif // CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H